From 5efb7384c1c175429ae81e709bc31de19222da73 Mon Sep 17 00:00:00 2001 From: Avaya Aggarwal <119044997+OnePunchMonk@users.noreply.github.com> Date: Wed, 14 Oct 2026 08:26:09 +0530 Subject: [PATCH] [diffusion] comfyui: consolidate 6 unrelated plugin fixes (#43493) Co-authored-by: Mick Co-authored-by: niehen6174 <1639206518@qq.com> --- .../ComfyUI_SGLDiffusion/core/generator.py | 7 +- .../ComfyUI_SGLDiffusion/core/server_api.py | 11 ++- .../executors/qwen_image.py | 14 ++-- .../apps/ComfyUI_SGLDiffusion/nodes.py | 9 ++- .../apps/ComfyUI_SGLDiffusion/utils.py | 80 +++++++++++++++++-- .../minimax_h3/stages/comfyui_step.py | 7 +- .../test/unit/test_comfyui_adapters.py | 38 +++++++++ .../test/unit/test_comfyui_generator.py | 24 ++++++ .../test/unit/test_comfyui_h3.py | 32 ++++++++ 9 files changed, 200 insertions(+), 22 deletions(-) diff --git a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/core/generator.py b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/core/generator.py index 35498cc570..327bb74e28 100644 --- a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/core/generator.py +++ b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/core/generator.py @@ -362,7 +362,7 @@ class SGLDiffusionGenerator: """Rebuild the worker if this cached executor no longer owns it.""" if self._owns_live(executor): return - reload = getattr(executor, "_sgld_reload", None) + reload = executor._sgld_reload if not reload: raise RuntimeError( "SGLD worker is down and this executor has no reload spec" @@ -373,9 +373,12 @@ class SGLDiffusionGenerator: ) self.load_model(**reload) executor.generator = self.generator - lora = getattr(executor, "_lora_input", None) + lora = executor._lora_input if lora and lora.get("lora_nickname"): executor.set_lora(**lora) + # The new worker's session cache starts empty, so conditioning the + # old worker already had cannot be restored; resend it. + executor.begin_sampler_run() def kill_generator(self): """Force-stop workers this owner started. Do not scan the process table.""" diff --git a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/core/server_api.py b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/core/server_api.py index 7e90f51fb2..33e287dd39 100644 --- a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/core/server_api.py +++ b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/core/server_api.py @@ -49,13 +49,16 @@ class SGLDiffusionServerAPI: - vae_precision: VAE model precision """ try: - # Remove /v1 from base_url for /models endpoint - models_url = self.base_url.removesuffix("/v1") + "/models" - response = requests.get(models_url, headers=self.headers, timeout=30) + response = requests.get( + f"{self.base_url}/models", headers=self.headers, timeout=30 + ) response.raise_for_status() - return response.json() + data = response.json() + return data["data"][0] except requests.exceptions.RequestException as e: raise RuntimeError(f"Failed to get model info: {str(e)}") + except (KeyError, IndexError) as e: + raise RuntimeError(f"Unexpected response shape from /models: {str(e)}") def generate_image( self, diff --git a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/executors/qwen_image.py b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/executors/qwen_image.py index c1a625c61c..e8bb5da272 100644 --- a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/executors/qwen_image.py +++ b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/executors/qwen_image.py @@ -1,6 +1,7 @@ """Qwen-Image adapters for the ComfyUI DiT-forward contract.""" import comfy.ldm.common_dit +import torch from .adapter import ComfyUIModelAdapter, PackedForward from .base import SGLDiffusionExecutor @@ -89,11 +90,14 @@ class QwenImageEditAdapter(QwenImageAdapter): ) -> PackedForward: packed = super().pack(x, timestep, context, **kwargs) if ref_latents: - pack_ref, orig_ref_shape = self._pack_latents(ref_latents[0]) - packed.extra_req["image_latent"] = pack_ref - packed.extra_req["vae_image_sizes"] = [ - (orig_ref_shape[-1], orig_ref_shape[-2]) - ] + packed_refs = [] + vae_image_sizes = [] + for ref_latent in ref_latents: + pack_ref, orig_ref_shape = self._pack_latents(ref_latent) + packed_refs.append(pack_ref) + vae_image_sizes.append((orig_ref_shape[-1], orig_ref_shape[-2])) + packed.extra_req["image_latent"] = torch.cat(packed_refs, dim=1) + packed.extra_req["vae_image_sizes"] = vae_image_sizes return packed diff --git a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/nodes.py b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/nodes.py index f86d4dbbee..64084e1313 100644 --- a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/nodes.py +++ b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/nodes.py @@ -25,6 +25,7 @@ from .utils import ( convert_video_to_comfy_video, get_image_path, is_empty_image, + resolve_video_path, ) @@ -663,7 +664,9 @@ class SGLDiffusionGenerateVideo: # Call API try: response = sgld_client.generate_video(**request_params) - video_path = response.get("file_path", "") + video_path = resolve_video_path( + response.get("file_path"), response.get("url") + ) video = convert_video_to_comfy_video(video_path, height, width) except Exception as e: raise RuntimeError(f"Failed to generate video: {str(e)}") @@ -871,10 +874,12 @@ class SGLDiffusionGenerateH3: try: response = sgld_client.generate_video(**request_params) + video_path = resolve_video_path( + response.get("file_path"), response.get("url") + ) except Exception as e: raise RuntimeError(f"Failed to generate MiniMax-H3 video: {str(e)}") - video_path = response.get("file_path", "") # H3 aligns the canvas server-side, so the resolved size is only known # from the response; short_edge and aspect_ratio cannot reconstruct it resolved_size = response.get("size", "") diff --git a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/utils.py b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/utils.py index 093b901e47..6d7da22501 100644 --- a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/utils.py +++ b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/utils.py @@ -2,11 +2,13 @@ import base64 import io import os import shutil +import subprocess import time import uuid import folder_paths import numpy as np +import requests import torch from comfy_api.input import VideoInput from PIL import Image @@ -156,14 +158,78 @@ class SGLDVideoInput(VideoInput): """ Abstract method to save the video input to a file. """ + if not os.path.exists(self.video_path): + raise FileNotFoundError( + f"Video not found at '{self.video_path}'. This file must be " + "readable from the ComfyUI process; a path on a remote " + "SGLang Diffusion server is not." + ) save_path = path - # Copy video file from video_path to save_path - if os.path.exists(self.video_path): - # Ensure destination directory exists - save_dir = os.path.dirname(save_path) - if save_dir: - os.makedirs(save_dir, exist_ok=True) - shutil.copy2(self.video_path, save_path) + save_dir = os.path.dirname(save_path) + if save_dir: + os.makedirs(save_dir, exist_ok=True) + shutil.copy2(self.video_path, save_path) + + def as_trimmed( + self, + start_time: float | None = None, + duration: float | None = None, + strict_duration: bool = False, + ): + """ + Required by VideoInput since ComfyUI #12107. + + Trims via ffmpeg stream-copy (fast, no re-encode); `strict_duration` + is accepted for interface compatibility but not enforced to frame + accuracy, since this node never needs sub-frame precision. + """ + start_time = start_time or 0.0 + if duration is not None and duration < 0: + return None + + temp_dir = folder_paths.get_temp_directory() + _ensure_dir(temp_dir) + ext = os.path.splitext(self.video_path)[1] or ".mp4" + dest = os.path.join(temp_dir, f"sgl_video_trim_{uuid.uuid4().hex[:8]}{ext}") + + cmd = ["ffmpeg", "-y", "-ss", str(start_time), "-i", self.video_path] + if duration is not None: + cmd += ["-t", str(duration)] + cmd += ["-c", "copy", dest] + subprocess.run(cmd, capture_output=True, check=True) + + return SGLDVideoInput(dest, self.height, self.width) + + +def resolve_video_path(file_path: str | None, url: str | None) -> str: + """ + Resolve the generated video to a path this process can use. + + The server sets `file_path` to None once a cloud upload succeeds (the + local file is removed); fetch `url` into the ComfyUI temp directory in + that case. Otherwise `file_path` is returned as-is; a server on another + host without cloud storage returns a path this process can't read, and + `SGLDVideoInput.save_to` raises clearly when that path turns out to be + unreadable. + """ + if file_path: + return file_path + if not url: + raise RuntimeError( + "Generated video has no local file_path and no url to fetch it from." + ) + + temp_dir = folder_paths.get_temp_directory() + _ensure_dir(temp_dir) + ext = os.path.splitext(url.split("?", 1)[0])[1] or ".mp4" + dest = os.path.join(temp_dir, f"sgl_video_{uuid.uuid4().hex[:8]}{ext}") + + with requests.get(url, timeout=120, stream=True) as response: + response.raise_for_status() + with open(dest, "wb") as f: + for chunk in response.iter_content(chunk_size=8192): + f.write(chunk) + return dest def convert_video_to_comfy_video( diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/comfyui_step.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/comfyui_step.py index 2a6e52a7b9..2989e51599 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/comfyui_step.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/comfyui_step.py @@ -193,11 +193,14 @@ def _layout_signature_matches( if len(values) < 5: return False text_len, latent_t, latent_h, latent_w, audio_t = values + _, patch_h, patch_w = _PATCH_SIZE + padded_h = (int(video_x.shape[3]) + patch_h - 1) // patch_h * patch_h + padded_w = (int(video_x.shape[4]) + patch_w - 1) // patch_w * patch_w return ( text_len == int(text.shape[0]) and latent_t == int(video_x.shape[2]) - and latent_h == int(video_x.shape[3]) - and latent_w == int(video_x.shape[4]) + and latent_h == padded_h + and latent_w == padded_w and audio_t == int(audio_x.shape[-1]) ) diff --git a/python/sglang/multimodal_gen/test/unit/test_comfyui_adapters.py b/python/sglang/multimodal_gen/test/unit/test_comfyui_adapters.py index 1afcff814e..acaaa61492 100644 --- a/python/sglang/multimodal_gen/test/unit/test_comfyui_adapters.py +++ b/python/sglang/multimodal_gen/test/unit/test_comfyui_adapters.py @@ -1,6 +1,9 @@ # SPDX-License-Identifier: Apache-2.0 """Pack / unpack contract for ComfyUI model adapters.""" +import importlib +import sys +import types from types import SimpleNamespace from unittest.mock import patch @@ -234,6 +237,41 @@ def test_comfyui_weight_access_gets_a_clear_error() -> None: base.state_dict() +def _qwen_image_edit_adapter(): + """Import the Qwen adapters with the one ComfyUI helper they use stubbed.""" + common_dit = types.ModuleType("comfy.ldm.common_dit") + common_dit.pad_to_patch_size = lambda x, patch_size: x + comfy, comfy_ldm = types.ModuleType("comfy"), types.ModuleType("comfy.ldm") + comfy.ldm, comfy_ldm.common_dit = comfy_ldm, common_dit + stubs = {"comfy": comfy, "comfy.ldm": comfy_ldm, "comfy.ldm.common_dit": common_dit} + with patch.dict(sys.modules, stubs): + module = importlib.import_module( + "sglang.multimodal_gen.apps.ComfyUI_SGLDiffusion.executors.qwen_image" + ) + return module.QwenImageEditAdapter() + + +def test_qwen_image_edit_pack_keeps_all_ref_latents() -> None: + """Qwen-Image-Edit packed only the first of several reference images.""" + adapter = _qwen_image_edit_adapter() + x = torch.ones(1, 16, 1, 90, 160) + timestep = torch.tensor([1.0]) + context = torch.ones(1, 19, 2560) + ref_a = torch.ones(1, 16, 1, 64, 64) + ref_b = torch.ones(1, 16, 1, 32, 96) + ref_c = torch.ones(1, 16, 1, 48, 48) + + packed = adapter.pack(x, timestep, context, ref_latents=[ref_a, ref_b, ref_c]) + + sizes = packed.extra_req["vae_image_sizes"] + assert sizes == [(64, 64), (96, 32), (48, 48)] + + expected_tokens = sum( + (h // 2) * (w // 2) for h, w in [(64, 64), (32, 96), (48, 48)] + ) + assert packed.extra_req["image_latent"].shape[1] == expected_tokens + + class _RowRecordingExecutor(SGLDiffusionExecutor): """Real forward/pack/unpack; the worker round trip just records the request.""" diff --git a/python/sglang/multimodal_gen/test/unit/test_comfyui_generator.py b/python/sglang/multimodal_gen/test/unit/test_comfyui_generator.py index d6a08037a3..d621c0931c 100644 --- a/python/sglang/multimodal_gen/test/unit/test_comfyui_generator.py +++ b/python/sglang/multimodal_gen/test/unit/test_comfyui_generator.py @@ -37,6 +37,7 @@ def test_ensure_rebuilds_when_another_model_owns_the_worker() -> None: stale = object() fresh = object() loads = [] + sampler_runs = [] executor = SimpleNamespace( generator=stale, _sgld_reload={ @@ -45,6 +46,7 @@ def test_ensure_rebuilds_when_another_model_owns_the_worker() -> None: "sgld_options": {}, }, _lora_input=None, + begin_sampler_run=lambda: sampler_runs.append(True), ) def fake_load(**kwargs): @@ -56,6 +58,7 @@ def test_ensure_rebuilds_when_another_model_owns_the_worker() -> None: runtime.ensure_executor(executor) assert loads == [executor._sgld_reload] assert executor.generator is fresh + assert sampler_runs == [True] def test_ensure_is_noop_when_executor_still_owns_live_worker() -> None: @@ -366,3 +369,24 @@ def test_renamed_h3_paths_resolve_through_the_plugin_model_id( assert seen["warmup_mode"] == "off" info = _get_config_info(path, model_id=seen["model_id"]) assert info.pipeline_config_cls.__name__ == config_cls + + +def test_get_model_info_reads_the_card_from_v1_models(monkeypatch) -> None: + """The plugin reads the OpenAI /v1/models route, not the deprecated root + /models, so it must unwrap the card from the list envelope.""" + from sglang.multimodal_gen.apps.ComfyUI_SGLDiffusion.core import server_api + + card = {"id": "wan", "object": "model", "task_type": "T2V"} + urls = [] + + def fake_get(url, headers=None, timeout=None): + urls.append(url) + return SimpleNamespace( + raise_for_status=lambda: None, + json=lambda: {"object": "list", "data": [card]}, + ) + + monkeypatch.setattr(server_api.requests, "get", fake_get) + client = server_api.SGLDiffusionServerAPI(base_url="http://host:30010/v1") + assert client.get_model_info() == card + assert urls == ["http://host:30010/v1/models"] diff --git a/python/sglang/multimodal_gen/test/unit/test_comfyui_h3.py b/python/sglang/multimodal_gen/test/unit/test_comfyui_h3.py index 3512b63319..98a8a376b5 100644 --- a/python/sglang/multimodal_gen/test/unit/test_comfyui_h3.py +++ b/python/sglang/multimodal_gen/test/unit/test_comfyui_h3.py @@ -84,6 +84,9 @@ class ComfyUIPackedLayout: def __init__( self, text_len, latent_t, latent_h, latent_w, audio_t, keyframes=None, refs=None ): + # ComfyUI builds PackedLayout from its patch-padded latent dimensions. + latent_h = (latent_h + 1) // 2 * 2 + latent_w = (latent_w + 1) // 2 * 2 frame, w_grid = _frame_grid(latent_h, latent_w) frame_rows = frame.shape[0] @@ -825,6 +828,35 @@ def test_keyframes_with_serialized_layout_build_kwargs(): assert branch.video_target_start == 4 +def test_odd_latent_keyframes_keep_comfyui_layout(): + video = torch.randn(1, 24, 2, 5, 7) + audio = torch.randn(1, 32, 2, 3) + context = torch.randn(1, 8, 16) + timestep = torch.tensor([500.0]) + sample_sigmas = torch.tensor([1.0, 0.5, 0.0]) + kf_latent = torch.randn(1, 24, 1, 5, 7) + keyframes = [{"resolved_frame_index": 0, "latent": kf_latent}] + layout = ComfyUIPackedLayout(8, 2, 5, 7, 3, keyframes=keyframes) + + inputs = comfyui_payload_to_branch_inputs( + video, + audio, + context, + minimax_payload={ + "keyframes": keyframes, + "cond_video_latents": [kf_latent], + "visual_cond_noise_aug": 1.0, + }, + sample_sigmas=sample_sigmas, + timestep=timestep, + layout=serialize_comfyui_layout(layout), + ) + + assert tuple(layout.signature[2:4]) == (6, 8) + assert inputs["padded_video_shape"][-2:] == (6, 8) + assert inputs["video_rows"].shape[0] == 12 + 2 * 3 * 4 + + def test_uniform_denoise_mask_scales_video_timestep(): video, audio, context, timestep, sample_sigmas = _t2va_tensors() inputs = comfyui_payload_to_branch_inputs(