[Feat] comfyui: video-to-video input and prompt enhancement for server-mode nodes (#43673)

This commit is contained in:
Avaya Aggarwal
2026-10-23 23:09:30 +08:00
committed by GitHub
parent c677171c1d
commit efd2ce8a9d
7 changed files with 1227 additions and 76 deletions
@@ -34,7 +34,8 @@ Connect to a standalone SGLang Diffusion server.
2. **Connect to Server**: Use the `SGLDiffusion Server Model` node to connect (default: `http://localhost:3000/v1`).
3. **Generate Content**:
- `SGLDiffusion Generate Image`: For text-to-image and image editing. Set `num_images` (1 to 10) to get several images from one request as one `IMAGE` batch, in order. Image `i` uses `seed + i`, so the same settings and seed reproduce the same batch; a seed of `-1` lets the server pick a random seed.
- `SGLDiffusion Generate Video`: For text-to-video and image-to-video.
- `SGLDiffusion Generate Video`: For text-to-video, image-to-video, and video-to-video (connect a `VIDEO`).
- Both nodes have an `enhance_prompt` toggle (the server needs `--prompt-enhancer-config`); the rewritten prompt comes out as `revised_prompt`.
- `SGLDiffusion Generate MiniMax-H3`: For joint video-and-audio on a standalone server.
4. **LoRA Support**: Use `SGLDiffusion Server Set LoRA` and `SGLDiffusion Server Unset LoRA`.
@@ -3,24 +3,43 @@ SGLang Diffusion Server API client.
Provides a low-level interface for interacting with SGLang Diffusion HTTP server.
"""
import contextlib
import json
import os
import re
import time
from typing import Any, Dict, Optional
from typing import Any, Callable, Dict, Optional
import requests
# Server-supplied names used to build a local path for a downloaded video.
_SAFE_VIDEO_ID = re.compile(r"[A-Za-z0-9_-]+")
_VIDEO_EXTENSIONS = (".mp4", ".webm", ".mov", ".mkv")
_MAX_VIDEO_BYTES = 2 * 1024**3
_DOWNLOAD_DEADLINE_S = 600
class SGLDiffusionServerAPI:
"""Client for SGLang Diffusion HTTP server API."""
def __init__(self, base_url: str, api_key: str = "sk-proj-1234567890"):
def __init__(
self,
base_url: str,
api_key: str = "sk-proj-1234567890",
image_timeout: int = 300,
):
"""
Initialize the API client.
Args:
base_url: Base URL of the SGLang Diffusion server (e.g., "http://localhost:30010/v1")
api_key: API key for authentication (default: "sk-proj-1234567890")
image_timeout: Seconds to wait for an image generate/edit request
before timing out. A large job (big size, or a first request
that triggers torch.compile/warmup) can need more than the
previous hardcoded 300s.
"""
self.image_timeout = image_timeout
# Ensure base_url doesn't end with /v1 if it's already there
if base_url.endswith("/v1"):
self.base_url = base_url
@@ -80,6 +99,7 @@ class SGLDiffusionServerAPI:
background: Optional[str] = "auto",
output_format: Optional[str] = None,
generator_device: Optional[str] = "cuda",
enhance_prompt: bool = False,
) -> Dict[str, Any]:
"""
Generate or edit an image using SGLang Diffusion API.
@@ -105,6 +125,7 @@ class SGLDiffusionServerAPI:
background: Background type ("auto", "transparent", "opaque")
output_format: Output format ("png", "jpeg", "webp")
generator_device: Device for random generator ("cuda" or "cpu")
enhance_prompt: Rewrite the prompt on the server first
Returns:
Dictionary containing the API response with generated/edited image data
@@ -133,6 +154,7 @@ class SGLDiffusionServerAPI:
background=background,
output_format=output_format,
generator_device=generator_device,
enhance_prompt=enhance_prompt,
)
# If image_path is provided, use edit endpoint
@@ -172,7 +194,7 @@ class SGLDiffusionServerAPI:
files=files,
data=data,
headers=headers,
timeout=300, # 5 minutes timeout for generation
timeout=self.image_timeout,
)
response.raise_for_status()
return response.json()
@@ -196,7 +218,7 @@ class SGLDiffusionServerAPI:
f"{self.base_url}/images/generations",
json=payload,
headers=self.headers,
timeout=300, # 5 minutes timeout for generation
timeout=self.image_timeout,
)
response.raise_for_status()
return response.json()
@@ -219,12 +241,19 @@ class SGLDiffusionServerAPI:
enable_teacache: bool = False,
generator_device: Optional[str] = "cuda",
input_reference: Optional[str] = None,
video_path: Optional[str] = None,
output_path: Optional[str] = None,
extra_fields: Optional[Dict[str, Any]] = None,
progress_callback: Optional[Callable[[int], None]] = None,
check_interrupt: Optional[Callable[[], None]] = None,
) -> Dict[str, Any]:
"""
Generate a video using SGLang Diffusion API and wait for completion.
``progress_callback`` gets the job's ``progress`` (0-100) on each poll.
``check_interrupt`` is called between polls and may raise to stop
waiting; the server has no abort yet, so the job itself keeps running.
Args:
prompt: Text prompt for video generation
size: Video size in format "WIDTHxHEIGHT" (e.g., "1280x720")
@@ -240,6 +269,8 @@ class SGLDiffusionServerAPI:
enable_teacache: Enable TEA cache acceleration
generator_device: Device for random generator ("cuda" or "cpu")
input_reference: Path to input reference image for image-to-video
video_path: Source clip for video-to-video. A local file is
uploaded; a URL or server-side path is sent as is
extra_fields: Model-specific request fields merged into the payload
last, so a caller can reach a model's own request surface
(MiniMax-H3's `task`/`conditions`/`target`, per-model flow
@@ -258,48 +289,89 @@ class SGLDiffusionServerAPI:
else:
size = "720x1280"
# Prepare request payload
payload: Dict[str, Any] = {
# Prepare request fields shared by the JSON and multipart paths.
# `output_path` is deliberately never sent: it used to be the
# ComfyUI node's own temp directory, which only exists on this
# machine and breaks as soon as the server runs elsewhere. Letting
# the server pick its own output location and fetching the result
# (see below) works for both the local and remote case.
fields: Dict[str, Any] = {
"prompt": prompt,
"size": size,
}
# Add optional parameters
if seconds is not None:
payload["seconds"] = seconds
fields["seconds"] = seconds
if fps is not None:
payload["fps"] = fps
fields["fps"] = fps
if num_frames is not None:
payload["num_frames"] = num_frames
fields["num_frames"] = num_frames
if negative_prompt:
payload["negative_prompt"] = negative_prompt
fields["negative_prompt"] = negative_prompt
if guidance_scale is not None:
payload["guidance_scale"] = guidance_scale
fields["guidance_scale"] = guidance_scale
if num_inference_steps is not None:
payload["num_inference_steps"] = num_inference_steps
fields["num_inference_steps"] = num_inference_steps
if seed is not None and seed >= 0:
payload["seed"] = seed
fields["seed"] = seed
if enable_teacache:
payload["enable_teacache"] = True
fields["enable_teacache"] = True
if generator_device:
payload["generator_device"] = generator_device
if input_reference:
payload["input_reference"] = input_reference
if output_path:
payload["output_path"] = output_path
fields["generator_device"] = generator_device
# merged last so a model-specific field wins over a generic default of
# the same name (H3 sizes its output from `target`, not `size`)
if extra_fields:
payload.update(extra_fields)
fields.update(extra_fields)
# Upload local files instead of sending their paths: a server on
# another machine cannot read this machine's filesystem (the images
# endpoint already does this for image_path). The server takes an
# uploaded source clip as `video_reference`.
string_fields: Dict[str, Any] = {}
uploads: Dict[str, str] = {}
for form_key, json_key, path in (
("input_reference", "input_reference", input_reference),
("video_reference", "video_path", video_path),
):
if not path:
continue
if self._is_local_file(path):
uploads[form_key] = path
else:
string_fields[json_key] = path
try:
# Create video generation job
response = requests.post(
f"{self.base_url}/videos",
json=payload,
headers=self.headers,
timeout=30,
)
if uploads:
# JSON, not repr: the server json-decodes form values, so a
# nested dict or False must not arrive as Python's repr
data = {
key: (
value if isinstance(value, (str, bytes)) else json.dumps(value)
)
for key, value in {**fields, **string_fields}.items()
}
with contextlib.ExitStack() as stack:
files = {
form_key: (
os.path.basename(path),
stack.enter_context(open(path, "rb")),
self._get_content_type(path),
)
for form_key, path in uploads.items()
}
response = requests.post(
f"{self.base_url}/videos",
files=files,
data=data,
headers={"Authorization": f"Bearer {self.api_key}"},
timeout=30,
)
else:
response = requests.post(
f"{self.base_url}/videos",
json={**fields, **string_fields},
headers=self.headers,
timeout=30,
)
response.raise_for_status()
video_job = response.json()
video_id = video_job.get("id")
@@ -312,6 +384,8 @@ class SGLDiffusionServerAPI:
start_time = time.time()
while time.time() - start_time < max_wait_time:
if check_interrupt:
check_interrupt()
try:
status_response = requests.get(
f"{self.base_url}/videos/{video_id}",
@@ -324,8 +398,13 @@ class SGLDiffusionServerAPI:
# Reset error counter on successful request
consecutive_errors = 0
if progress_callback and status.get("progress") is not None:
progress_callback(int(status["progress"]))
if status.get("status") == "completed":
return status
# Localize outside this try, so a failed download
# fails fast instead of being retried as a poll error.
break
elif status.get("status") == "failed":
error = status.get("error", {})
error_msg = (
@@ -350,14 +429,80 @@ class SGLDiffusionServerAPI:
f"Network error after {consecutive_errors} consecutive failures: {str(e)}"
)
time.sleep(poll_interval)
raise TimeoutError(
f"Video generation timed out after {max_wait_time} seconds"
)
# Sleep in short slices so Cancel is honoured within ~1s.
for _ in range(poll_interval):
if check_interrupt:
check_interrupt()
time.sleep(1)
else:
raise TimeoutError(
f"Video generation timed out after {max_wait_time} seconds"
)
return self._localize_video_result(status, video_id, output_path)
except requests.exceptions.RequestException as e:
raise RuntimeError(f"Failed to generate video: {str(e)}")
def _localize_video_result(
self, status: Dict[str, Any], video_id: str, output_path: Optional[str]
) -> Dict[str, Any]:
"""Make sure ``file_path`` in a completed job points at a local file.
The server's own ``file_path`` is only readable when the server runs
on this machine. When it isn't (no such local file), download the
result through ``GET /videos/{id}/content`` instead of trusting a
path that belongs to a different filesystem. A job uploaded to cloud
storage has no file_path and a ``url`` instead; ``/content`` refuses
those, so it is returned as is and the node fetches the url.
"""
file_path = status.get("file_path")
if file_path and os.path.exists(file_path):
return status
if not file_path and status.get("url"):
return status
if not isinstance(video_id, str) or not _SAFE_VIDEO_ID.fullmatch(video_id):
raise RuntimeError(f"Refusing to save video with unsafe id {video_id!r}")
ext = os.path.splitext(file_path or "")[1].lower() or ".mp4"
if ext not in _VIDEO_EXTENSIONS:
ext = ".mp4"
target_dir = os.path.realpath(output_path or os.getcwd())
os.makedirs(target_dir, exist_ok=True)
local_path = os.path.realpath(os.path.join(target_dir, f"{video_id}{ext}"))
if os.path.commonpath([target_dir, local_path]) != target_dir:
raise RuntimeError(f"Refusing to save video outside {target_dir}")
deadline = time.monotonic() + _DOWNLOAD_DEADLINE_S
response = requests.get(
f"{self.base_url}/videos/{video_id}/content",
headers=self.headers,
timeout=300,
stream=True,
)
try:
response.raise_for_status()
written = 0
with open(local_path, "wb") as fh:
for chunk in response.iter_content(chunk_size=1024 * 1024):
written += len(chunk)
if written > _MAX_VIDEO_BYTES:
raise RuntimeError(
f"Video download exceeded {_MAX_VIDEO_BYTES} bytes"
)
if time.monotonic() > deadline:
raise RuntimeError(
f"Video download exceeded {_DOWNLOAD_DEADLINE_S}s"
)
fh.write(chunk)
except BaseException:
with contextlib.suppress(OSError):
os.remove(local_path)
raise
finally:
close = getattr(response, "close", None)
if close is not None:
close()
status = dict(status)
status["file_path"] = local_path
return status
def _build_image_common_params(
self,
prompt: str,
@@ -372,6 +517,7 @@ class SGLDiffusionServerAPI:
background: Optional[str] = None,
output_format: Optional[str] = None,
generator_device: Optional[str] = None,
enhance_prompt: bool = False,
) -> Dict[str, Any]:
"""
Build common parameters for both image generation and editing.
@@ -403,9 +549,17 @@ class SGLDiffusionServerAPI:
params["output_format"] = output_format
if generator_device:
params["generator_device"] = generator_device
if enhance_prompt:
params["enhance_prompt"] = True
return params
@staticmethod
def _is_local_file(path: str) -> bool:
return not path.startswith(("http://", "https://", "data:")) and os.path.isfile(
path
)
def _get_content_type(self, file_path: str) -> str:
"""Get content type based on file extension."""
ext = os.path.splitext(file_path)[1].lower()
@@ -414,6 +568,10 @@ class SGLDiffusionServerAPI:
".jpg": "image/jpeg",
".jpeg": "image/jpeg",
".webp": "image/webp",
".mp4": "video/mp4",
".webm": "video/webm",
".mov": "video/quicktime",
".mkv": "video/x-matroska",
}
return content_types.get(ext, "image/png")
@@ -422,6 +580,7 @@ class SGLDiffusionServerAPI:
lora_nickname: str,
lora_path: Optional[str] = None,
target: str = "all",
strength: Optional[float] = None,
) -> Dict[str, Any]:
"""
Set a LoRA adapter for the specified transformer(s).
@@ -435,6 +594,8 @@ class SGLDiffusionServerAPI:
- "transformer": Apply only to the primary transformer (high noise for Wan2.2)
- "transformer_2": Apply only to transformer_2 (low noise for Wan2.2)
- "critic": Apply only to the critic model
strength: LoRA merge strength. The server defaults to 1.0 when
not given; values < 1.0 reduce the effect.
Returns:
Dictionary containing the API response with status and message
@@ -451,6 +612,8 @@ class SGLDiffusionServerAPI:
# Add optional lora_path if provided
if lora_path:
payload["lora_path"] = lora_path
if strength is not None:
payload["strength"] = strength
try:
response = requests.post(
@@ -3,9 +3,15 @@ ComfyUI nodes for SGLang Diffusion integration.
Provides nodes for connecting to SGLang Diffusion server and generating images/videos.
"""
import base64
import ipaddress
import os
import socket
import urllib.parse
import uuid
import comfy.model_management
import comfy.utils
import folder_paths
import torch
@@ -74,6 +80,15 @@ class SGLDH3DistilledSigmas:
return (torch.tensor(distilled_h3_sigmas(model_type)),)
def _comfy_job_hooks() -> dict:
"""Wire a server-mode job into ComfyUI's progress bar and Cancel button."""
pbar = comfy.utils.ProgressBar(100)
return {
"progress_callback": pbar.update_absolute,
"check_interrupt": comfy.model_management.throw_exception_if_processing_interrupted,
}
class SGLDLoraLoader:
@classmethod
def INPUT_TYPES(cls):
@@ -216,7 +231,21 @@ class SGLDiffusionServerModel:
"multiline": False,
},
),
}
},
"optional": {
"image_timeout": (
"INT",
{
"default": 300,
"min": 1,
"max": 3600,
"tooltip": "Seconds to wait for an image "
"generate/edit request (e.g. a large job, or "
"torch.compile warmup on the first request, can "
"need more than 300s)",
},
),
},
}
RETURN_TYPES = ("SGLD_CLIENT", "STRING")
@@ -224,9 +253,11 @@ class SGLDiffusionServerModel:
FUNCTION = "load_server"
CATEGORY = "SGLDiffusion"
def load_server(self, base_url: str, api_key: str):
def load_server(self, base_url: str, api_key: str, image_timeout: int = 300):
"""Initialize OpenAI client for SGLang Diffusion server."""
client = SGLDiffusionServerAPI(base_url=base_url, api_key=api_key)
client = SGLDiffusionServerAPI(
base_url=base_url, api_key=api_key, image_timeout=image_timeout
)
try:
model_info = client.get_model_info()
# Format model_info as a readable string
@@ -330,11 +361,19 @@ class SGLDiffusionGenerateImage:
"one IMAGE batch. Image i uses seed + i.",
},
),
"enhance_prompt": (
"BOOLEAN",
{
"default": False,
"tooltip": "Rewrite the prompt on the server (needs "
"--prompt-enhancer-config); returned as revised_prompt",
},
),
},
}
RETURN_TYPES = ("IMAGE",)
RETURN_NAMES = ("image",)
RETURN_TYPES = ("IMAGE", "STRING")
RETURN_NAMES = ("image", "revised_prompt")
FUNCTION = "generate_image"
CATEGORY = "SGLDiffusion"
OUTPUT_NODE = False
@@ -352,6 +391,7 @@ class SGLDiffusionGenerateImage:
height: int = 1024,
enable_teacache: bool = False,
num_images: int = 1,
enhance_prompt: bool = False,
):
"""Generate image using SGLang Diffusion API."""
if not positive_prompt:
@@ -382,6 +422,8 @@ class SGLDiffusionGenerateImage:
request_params["seed"] = seed
if enable_teacache:
request_params["enable_teacache"] = True
if enhance_prompt:
request_params["enhance_prompt"] = True
if image is not None:
# If the image is empty, use the size of the image to generate the image
if is_empty_image(image):
@@ -397,7 +439,9 @@ class SGLDiffusionGenerateImage:
except Exception as e:
raise RuntimeError(f"Failed to generate image: {str(e)}")
return (_decode_image_batch(response, num_images),)
images = _decode_image_batch(response, num_images)
revised_prompt = response["data"][0].get("revised_prompt") or positive_prompt
return (images, revised_prompt)
class SGLDiffusionGenerateVideo:
@@ -431,6 +475,10 @@ class SGLDiffusionGenerateVideo:
"tooltip": "input image to use for image-to-video",
},
),
"video": (
"VIDEO",
{"tooltip": "source clip for video-to-video"},
),
"seed": (
"INT",
{
@@ -508,11 +556,19 @@ class SGLDiffusionGenerateVideo:
"default": False,
},
),
"enhance_prompt": (
"BOOLEAN",
{
"default": False,
"tooltip": "Rewrite the prompt on the server (needs "
"--prompt-enhancer-config); returned as revised_prompt",
},
),
},
}
RETURN_TYPES = ("VIDEO", "STRING")
RETURN_NAMES = ("video", "video_path")
RETURN_TYPES = ("VIDEO", "STRING", "STRING")
RETURN_NAMES = ("video", "video_path", "revised_prompt")
FUNCTION = "generate_video"
CATEGORY = "SGLDiffusion"
OUTPUT_NODE = False
@@ -532,6 +588,8 @@ class SGLDiffusionGenerateVideo:
fps: int = 24,
seconds: int = 5,
enable_teacache: bool = False,
video=None,
enhance_prompt: bool = False,
):
"""Generate video using SGLang Diffusion API."""
if not positive_prompt:
@@ -570,18 +628,35 @@ class SGLDiffusionGenerateVideo:
request_params["size"] = size
else:
request_params["input_reference"] = get_image_path(image)
if video is not None:
request_params["video_path"] = os.path.join(
output_dir, f"sgl_input_{uuid.uuid4().hex[:8]}.mp4"
)
video.save_to(request_params["video_path"])
if enhance_prompt:
request_params["extra_fields"] = {"enhance_prompt": True}
# Call API
try:
response = sgld_client.generate_video(**request_params)
response = sgld_client.generate_video(
**request_params, **_comfy_job_hooks()
)
video_path = resolve_video_path(
response.get("file_path"), response.get("url")
)
# The server may have resolved a different size than requested
# (e.g. to match an image-to-video input's aspect ratio).
resolved_size = response.get("size", "")
if resolved_size:
width, height = (int(v) for v in resolved_size.split("x"))
video = convert_video_to_comfy_video(video_path, height, width)
except comfy.model_management.InterruptProcessingException:
raise
except Exception as e:
raise RuntimeError(f"Failed to generate video: {str(e)}")
return (video, video_path)
revised_prompt = response.get("revised_prompt") or positive_prompt
return (video, video_path, revised_prompt)
class SGLDiffusionGenerateH3:
@@ -668,11 +743,61 @@ class SGLDiffusionGenerateH3:
OUTPUT_NODE = False
@staticmethod
def _material_uri(value: str) -> str:
"""Local paths become file:// URIs; remote URLs are passed through."""
def _image_material_uri(image: torch.Tensor) -> str:
"""Embed the image inline as a base64 data URI.
A `file://` path only works when the SGLD server runs on this same
machine; the server's H3 material loader accepts `data:` URIs
directly (no upload endpoint needed), so this works locally and
remotely alike.
"""
path = get_image_path(image)
try:
with open(path, "rb") as fh:
encoded = base64.b64encode(fh.read()).decode("ascii")
finally:
os.remove(path)
return f"data:image/png;base64,{encoded}"
@staticmethod
def _is_local_server(base_url: str) -> bool:
host = (urllib.parse.urlparse(base_url).hostname or "").lower()
if not host:
return False
if host in ("localhost", "0.0.0.0", "::"):
return True
own = {socket.gethostname().lower(), socket.getfqdn().lower()}
if host in own:
return True
try:
infos = socket.getaddrinfo(host, None)
except (OSError, UnicodeError):
return False
addrs = {info[4][0].split("%", 1)[0] for info in infos}
return bool(addrs) and all(
ipaddress.ip_address(addr).is_loopback for addr in addrs
)
@classmethod
def _remote_material_uri(cls, value: str, kind: str, base_url: str) -> str:
"""Remote URLs pass through. A local path only works when the SGLD
server is also on this machine: H3's server-side `/v1/videos`
endpoint has no multipart slot for the video/audio conditions this
node sends through `extra_fields.conditions`, unlike the image
conditions above (which sidestep that by embedding bytes inline).
Raise clearly instead of building a `file://` URI that will
silently fail against any server that isn't local.
"""
# file:// already names a server-side path
if value.startswith(("http://", "https://", "file://")):
return value
return f"file://{os.path.abspath(value)}"
if cls._is_local_server(base_url):
return f"file://{os.path.abspath(value)}"
raise ValueError(
f"{kind} must be an http(s) URL when the SGLD server "
f"({base_url}) isn't on this machine; got a local path "
f"({value!r}), which that server has no way to read or upload."
)
def generate(
self,
@@ -705,7 +830,7 @@ class SGLDiffusionGenerateH3:
conditions.append(
{
"type": "image",
"uri": self._material_uri(get_image_path(first_frame)),
"uri": self._image_material_uri(first_frame),
"role": "keyframe",
"frame_index": 0,
}
@@ -714,7 +839,7 @@ class SGLDiffusionGenerateH3:
conditions.append(
{
"type": "image",
"uri": self._material_uri(get_image_path(last_frame)),
"uri": self._image_material_uri(last_frame),
"role": "keyframe",
"frame_index": -1,
}
@@ -723,7 +848,7 @@ class SGLDiffusionGenerateH3:
conditions.append(
{
"type": "image",
"uri": self._material_uri(get_image_path(reference_image)),
"uri": self._image_material_uri(reference_image),
"role": "reference",
}
)
@@ -731,7 +856,9 @@ class SGLDiffusionGenerateH3:
conditions.append(
{
"type": "video",
"uri": self._material_uri(reference_video),
"uri": self._remote_material_uri(
reference_video, "reference_video", sgld_client.base_url
),
"role": "reference",
}
)
@@ -739,7 +866,9 @@ class SGLDiffusionGenerateH3:
conditions.append(
{
"type": "audio",
"uri": self._material_uri(reference_audio),
"uri": self._remote_material_uri(
reference_audio, "reference_audio", sgld_client.base_url
),
"role": "reference",
}
)
@@ -783,10 +912,14 @@ class SGLDiffusionGenerateH3:
request_params["seed"] = seed
try:
response = sgld_client.generate_video(**request_params)
response = sgld_client.generate_video(
**request_params, **_comfy_job_hooks()
)
video_path = resolve_video_path(
response.get("file_path"), response.get("url")
)
except comfy.model_management.InterruptProcessingException:
raise
except Exception as e:
raise RuntimeError(f"Failed to generate MiniMax-H3 video: {str(e)}")
@@ -838,6 +971,17 @@ class SGLDiffusionServerSetLora:
"tooltip": "Which transformer(s) to apply the LoRA to",
},
),
"strength": (
"FLOAT",
{
"default": 1.0,
"min": 0.0,
"max": 10.0,
"step": 0.01,
"tooltip": "LoRA merge strength; the server's own "
"default is 1.0",
},
),
},
}
@@ -853,6 +997,7 @@ class SGLDiffusionServerSetLora:
lora_name: str = "",
lora_nickname: str = "",
target: str = "all",
strength: float = 1.0,
):
"""Set LoRA adapter using SGLang Diffusion API."""
if lora_nickname == "":
@@ -863,6 +1008,7 @@ class SGLDiffusionServerSetLora:
"lora_nickname": lora_nickname,
"lora_path": lora_name,
"target": target,
"strength": strength,
}
# Call API
@@ -5,7 +5,9 @@ only HTTP, so a change on either side of the payload contract fails here.
"""
import importlib.util
import os
import sys
import tempfile
import types
from pathlib import Path
from unittest import mock
@@ -125,25 +127,56 @@ class _Response:
return self._payload
_TMP_FILES = []
def _tmp_file(suffix):
"""A temp file path that the autouse fixture below deletes afterwards."""
with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as fh:
_TMP_FILES.append(fh.name)
return fh.name
@pytest.fixture(autouse=True)
def _remove_tmp_files():
yield
while _TMP_FILES:
path = _TMP_FILES.pop()
if os.path.exists(path):
os.remove(path)
def _run_node(**node_kwargs):
"""Drive the node through the real client and capture the POST payload."""
client = SGLDiffusionServerAPI(base_url="http://127.0.0.1:30010")
captured = {}
# A real, existing path: the client now checks the server's reported
# file_path with os.path.exists before trusting it (see
# SGLDiffusionServerAPI._localize_video_result), since on a remote
# server that path belongs to a different filesystem.
out_path = _tmp_file(".mp4")
def fake_post(url, json=None, headers=None, timeout=None):
captured.update(json)
def fake_post(url, json=None, headers=None, timeout=None, files=None, data=None):
captured.update(json or data or {})
return _Response({"id": "job-1"})
def fake_get(url, headers=None, timeout=None):
def fake_get(url, headers=None, timeout=None, **kwargs):
return _Response(
{
"id": "job-1",
"status": "completed",
"size": RESOLVED_SIZE,
"file_path": "/tmp/out.mp4",
"file_path": out_path,
}
)
def fake_get_image_path(image):
# _image_material_uri reads this file's bytes and then deletes it.
path = _tmp_file(".png")
with open(path, "wb") as fh:
fh.write(b"\x89PNG\r\n\x1a\n" + b"\x00" * 16)
return path
node = SGLDiffusionGenerateH3()
with (
mock.patch(
@@ -156,7 +189,7 @@ def _run_node(**node_kwargs):
),
mock.patch(
f"{PKG}.nodes.get_image_path",
side_effect=lambda image: "/tmp/frame.png",
side_effect=fake_get_image_path,
),
):
result = node.generate(sgld_client=client, **node_kwargs)
@@ -191,7 +224,11 @@ def test_fl2va_maps_keyframes_to_frame_indices():
assert [c["role"] for c in payload["conditions"]] == ["keyframe", "keyframe"]
assert [c["frame_index"] for c in payload["conditions"]] == [0, -1]
assert all(c["uri"].startswith("file:///") for c in payload["conditions"])
# Images are embedded inline (data: URI), not as a local file:// path
# the server may not be able to read.
assert all(
c["uri"].startswith("data:image/png;base64,") for c in payload["conditions"]
)
def test_ref2va_preserves_modality_order_for_prompt_tags():
@@ -222,7 +259,7 @@ def test_node_reports_the_server_resolved_canvas():
_, (video, video_path) = _run_node(positive_prompt="a cat", task="t2va")
assert video.get_dimensions() == (1344, 768)
assert video_path == "/tmp/out.mp4"
assert os.path.exists(video_path)
@pytest.mark.parametrize(
@@ -242,9 +279,10 @@ def test_extra_fields_win_over_generic_defaults():
"""A model's own field must not be shadowed by a same-named generic default."""
client = SGLDiffusionServerAPI(base_url="http://127.0.0.1:30010")
captured = {}
out_path = _tmp_file(".mp4")
def fake_post(url, json=None, headers=None, timeout=None):
captured.update(json)
def fake_post(url, json=None, headers=None, timeout=None, files=None, data=None):
captured.update(json or data or {})
return _Response({"id": "job-1"})
with (
@@ -255,7 +293,12 @@ def test_extra_fields_win_over_generic_defaults():
mock.patch(
f"{PKG}.core.server_api.requests.get",
side_effect=lambda *a, **k: _Response(
{"id": "job-1", "status": "completed", "size": RESOLVED_SIZE}
{
"id": "job-1",
"status": "completed",
"size": RESOLVED_SIZE,
"file_path": out_path,
}
),
),
):
@@ -0,0 +1,758 @@
"""Regression tests for ComfyUI_SGLDiffusion server-mode bugs found in audit.
Each bug gets a test that fails against the pre-fix code and passes after.
These stub ComfyUI/ torch-adjacent modules the same way test_h3_request.py
does, so they run without a ComfyUI install or a GPU.
"""
import base64
import importlib.util
import io
import os
import sys
import tempfile
import types
from pathlib import Path
from unittest import mock
import pytest
import torch
from PIL import Image
PLUGIN_DIR = Path(__file__).resolve().parents[1]
PKG = "sgld_server_mode_under_test"
def _install_comfy_stubs() -> None:
folder_paths = types.ModuleType("folder_paths")
_tmp = tempfile.mkdtemp(prefix="sgld_test_temp_")
folder_paths.get_temp_directory = lambda: _tmp
folder_paths._diffusion_models_dirs = []
folder_paths.folder_names_and_paths = {
"diffusion_models": ([], {".safetensors", ".sft"}),
"loras": ([], {".safetensors"}),
}
def get_filename_list(name):
names = set()
for folder in folder_paths.folder_names_and_paths.get(name, ([], set()))[0]:
if os.path.isdir(folder):
names.update(os.listdir(folder))
return sorted(names)
def get_folder_paths(name):
return list(folder_paths.folder_names_and_paths.get(name, ([], set()))[0])
def get_full_path(name, filename):
for folder in get_folder_paths(name):
candidate = os.path.join(folder, filename)
if os.path.isfile(candidate):
return candidate
return None
folder_paths.get_filename_list = get_filename_list
folder_paths.get_folder_paths = get_folder_paths
folder_paths.get_full_path = get_full_path
sys.modules["folder_paths"] = folder_paths
comfy_api = types.ModuleType("comfy_api")
comfy_api_input = types.ModuleType("comfy_api.input")
class VideoInput:
pass
comfy_api_input.VideoInput = VideoInput
comfy_api.input = comfy_api_input
sys.modules["comfy_api"] = comfy_api
sys.modules["comfy_api.input"] = comfy_api_input
# No comfy_api.input_impl: exercises the SGLDVideoInput fallback path.
sys.modules.pop("comfy_api.input_impl", None)
comfy = types.ModuleType("comfy")
comfy.model_detection = types.ModuleType("comfy.model_detection")
comfy.model_management = types.ModuleType("comfy.model_management")
comfy_utils = types.ModuleType("comfy.utils")
for name in (
"calculate_parameters",
"load_torch_file",
"state_dict_prefix_replace",
"unet_to_diffusers",
):
setattr(comfy_utils, name, lambda *a, **k: None)
class InterruptProcessingException(Exception):
pass
_interrupt = {"flag": False}
def throw_exception_if_processing_interrupted():
if _interrupt["flag"]:
raise InterruptProcessingException()
comfy.model_management.InterruptProcessingException = InterruptProcessingException
comfy.model_management.throw_exception_if_processing_interrupted = (
throw_exception_if_processing_interrupted
)
comfy.model_management._test_interrupt = _interrupt
class ProgressBar:
instances = []
def __init__(self, total):
self.total = total
self.values = []
ProgressBar.instances.append(self)
def update_absolute(self, value, total=None, preview=None):
self.values.append(value)
comfy_utils.ProgressBar = ProgressBar
comfy.utils = comfy_utils
comfy_model_patcher = types.ModuleType("comfy.model_patcher")
class ModelPatcher:
def __init__(self, model, load_device, offload_device, size=0, *a, **k):
self.model = model
self.load_device = load_device
self.offload_device = offload_device
self.size = size
self.patches = {}
self.object_patches = {}
self.model_options = {}
self.backup = {}
self.object_patches_backup = {}
self.wrappers = {}
self.callbacks = {}
self.patches_uuid = None
self.weight_inplace_update = False
def add_wrapper_with_key(self, *a, **k):
pass
# ComfyUI: loaded_size() reads the model's own weight counter (0 for
# SGLD, whose weights live in the worker); detach() is what
# LoadedModel.model_unload always ends in.
def loaded_size(self):
return getattr(self.model, "model_loaded_weight_memory", 0)
def detach(self, unpatch_all=True):
return self.model
comfy_model_patcher.ModelPatcher = ModelPatcher
comfy.model_patcher = comfy_model_patcher
comfy_patcher_extension = types.ModuleType("comfy.patcher_extension")
class WrappersMP:
SAMPLER_SAMPLE = "SAMPLER_SAMPLE"
comfy_patcher_extension.WrappersMP = WrappersMP
comfy.patcher_extension = comfy_patcher_extension
sys.modules["comfy"] = comfy
sys.modules["comfy.model_detection"] = comfy.model_detection
sys.modules["comfy.model_management"] = comfy.model_management
sys.modules["comfy.utils"] = comfy_utils
sys.modules["comfy.model_patcher"] = comfy_model_patcher
sys.modules["comfy.patcher_extension"] = comfy_patcher_extension
return folder_paths
def _load(module_name: str, relative_path: str):
spec = importlib.util.spec_from_file_location(
module_name, PLUGIN_DIR / relative_path
)
module = importlib.util.module_from_spec(spec)
sys.modules[module_name] = module
spec.loader.exec_module(module)
return module
def _load_plugin():
folder_paths = _install_comfy_stubs()
package = types.ModuleType(PKG)
package.__path__ = [str(PLUGIN_DIR)]
sys.modules[PKG] = package
core = types.ModuleType(f"{PKG}.core")
core.__path__ = [str(PLUGIN_DIR / "core")]
sys.modules[f"{PKG}.core"] = core
server_api = _load(f"{PKG}.core.server_api", "core/server_api.py")
core.SGLDiffusionServerAPI = server_api.SGLDiffusionServerAPI
model_patcher = _load(f"{PKG}.core.model_patcher", "core/model_patcher.py")
core.SGLDModelPatcher = model_patcher.SGLDModelPatcher
generator = _load(f"{PKG}.core.generator", "core/generator.py")
core.SGLDiffusionGenerator = generator.SGLDiffusionGenerator
_load(f"{PKG}.utils", "utils.py")
nodes = _load(f"{PKG}.nodes", "nodes.py")
return (
server_api,
model_patcher,
generator,
nodes,
sys.modules[f"{PKG}.utils"],
folder_paths,
)
SERVER_API, MODEL_PATCHER, GENERATOR, NODES, UTILS, FOLDER_PATHS = _load_plugin()
SGLDiffusionServerAPI = SERVER_API.SGLDiffusionServerAPI
SGLDModelPatcher = MODEL_PATCHER.SGLDModelPatcher
class _Response:
def __init__(self, payload, status_code=200):
self._payload = payload
self.status_code = status_code
def raise_for_status(self):
pass
def json(self):
return self._payload
def iter_content(self, chunk_size=None):
yield b"fake-mp4-bytes"
# --- Bug 9: hardcoded 300s timeout on image requests -----------------------
_TMP_FILES = []
def _tmp_file(suffix):
"""A temp file path that the autouse fixture below deletes afterwards."""
with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as fh:
_TMP_FILES.append(fh.name)
return fh.name
@pytest.fixture(autouse=True)
def _remove_tmp_files():
yield
while _TMP_FILES:
path = _TMP_FILES.pop()
if os.path.exists(path):
os.remove(path)
def test_image_timeout_is_configurable():
client = SGLDiffusionServerAPI(base_url="http://127.0.0.1:1234", image_timeout=900)
captured = {}
def fake_post(url, json=None, headers=None, timeout=None, **kwargs):
captured["timeout"] = timeout
return _Response({"data": [{"b64_json": "x"}]})
with mock.patch(f"{PKG}.core.server_api.requests.post", side_effect=fake_post):
client.generate_image(prompt="a cat")
assert captured["timeout"] == 900
# --- Bug 10: batched image inputs were silently truncated to image[0] ------
def test_get_image_path_uses_first_image_of_a_batch_with_a_warning(caplog):
batch = torch.zeros(3, 8, 8, 3)
with caplog.at_level("WARNING"):
path = UTILS.get_image_path(batch)
assert os.path.exists(path)
os.remove(path)
assert "batch of 3 images" in caplog.text
def test_get_image_path_accepts_single_image():
single = torch.zeros(1, 8, 8, 3)
path = UTILS.get_image_path(single)
assert os.path.exists(path)
os.remove(path)
# --- Bug 11: LoRA strength could not be set in server mode -----------------
def test_set_lora_sends_strength_to_server():
client = SGLDiffusionServerAPI(base_url="http://127.0.0.1:1234")
captured = {}
def fake_post(url, json=None, headers=None, timeout=None, **kwargs):
captured.update(json)
return _Response({"status": "ok"})
with mock.patch(f"{PKG}.core.server_api.requests.post", side_effect=fake_post):
client.set_lora(
lora_nickname="style", lora_path="style.safetensors", strength=0.7
)
assert captured["strength"] == 0.7
def test_set_lora_node_exposes_strength(monkeypatch):
client = SGLDiffusionServerAPI(base_url="http://127.0.0.1:1234")
node = NODES.SGLDiffusionServerSetLora()
captured = {}
def fake_set_lora(**kwargs):
captured.update(kwargs)
return {"status": "ok"}
monkeypatch.setattr(client, "set_lora", fake_set_lora)
node.set_lora(sgld_client=client, lora_name="style.safetensors", strength=0.5)
assert captured["strength"] == 0.5
# --- Bug 6/7: SGLDVideoInput didn't implement VideoInput; dims could be wrong
def test_video_input_fallback_fails_loudly_instead_of_returning_wrong_data():
video = UTILS.convert_video_to_comfy_video("/tmp/out.mp4", height=720, width=1280)
assert isinstance(video, UTILS.SGLDVideoInput) # no comfy_api.input_impl stubbed
with pytest.raises(NotImplementedError):
video.get_components()
def test_video_input_save_to_refuses_silent_format_drop():
video = UTILS.SGLDVideoInput("/tmp/out.mp4", height=720, width=1280)
with pytest.raises(NotImplementedError):
video.save_to("/tmp/converted.webm", format="webm")
def test_generic_video_node_reports_server_resolved_size_not_requested_size():
client = SGLDiffusionServerAPI(base_url="http://127.0.0.1:1234")
node = NODES.SGLDiffusionGenerateVideo()
with (
mock.patch.object(
client,
"generate_video",
return_value={"file_path": "/tmp/x.mp4", "size": "1280x704"},
),
):
video, _, _ = node.generate_video(
sgld_client=client,
positive_prompt="a cat",
width=1280,
height=720, # requested 720, server actually resolved to 704
)
assert video.get_dimensions() == (1280, 704)
# --- Bug 3: server-mode nodes sent local paths a remote server can't read --
def test_generate_video_uploads_local_input_reference_instead_of_sending_path():
client = SGLDiffusionServerAPI(base_url="http://127.0.0.1:1234")
local_image = _tmp_file(".png")
with open(local_image, "wb") as fh:
fh.write(b"fake-png-bytes")
seen = {}
def fake_post(url, json=None, headers=None, timeout=None, files=None, data=None):
seen["files"] = files
seen["data"] = data
seen["json"] = json
return _Response({"id": "job-1"})
out_path = _tmp_file(".mp4")
def fake_get(url, headers=None, timeout=None, **kwargs):
return _Response({"id": "job-1", "status": "completed", "file_path": out_path})
with (
mock.patch(f"{PKG}.core.server_api.requests.post", side_effect=fake_post),
mock.patch(f"{PKG}.core.server_api.requests.get", side_effect=fake_get),
):
client.generate_video(prompt="a cat", input_reference=local_image)
os.remove(local_image)
# A local path must be uploaded as file bytes, not sent as a path string
# the server (possibly on another machine) has no way to read.
assert seen["files"] is not None
assert seen["json"] is None
def _capture_video_request(client, **kwargs):
seen = {}
def fake_post(url, json=None, headers=None, timeout=None, files=None, data=None):
seen["files"] = files and {
key: (name, fh.read(), content_type)
for key, (name, fh, content_type) in files.items()
}
seen["data"] = data
seen["json"] = json
return _Response({"id": "job-1"})
out_path = _tmp_file(".mp4")
def fake_get(url, headers=None, timeout=None, **kw):
return _Response({"id": "job-1", "status": "completed", "file_path": out_path})
with (
mock.patch(f"{PKG}.core.server_api.requests.post", side_effect=fake_post),
mock.patch(f"{PKG}.core.server_api.requests.get", side_effect=fake_get),
):
client.generate_video(prompt="a cat", **kwargs)
return seen
def test_generate_video_uploads_local_source_video_as_video_reference():
client = SGLDiffusionServerAPI(base_url="http://127.0.0.1:1234")
with tempfile.TemporaryDirectory() as tmp:
clip = os.path.join(tmp, "clip.mp4")
image = os.path.join(tmp, "ref.png")
for path, body in ((clip, b"fake-mp4"), (image, b"fake-png")):
with open(path, "wb") as fh:
fh.write(body)
seen = _capture_video_request(client, video_path=clip, input_reference=image)
# Both local files go up as multipart; neither path string is sent.
assert seen["json"] is None
assert seen["files"]["video_reference"] == ("clip.mp4", b"fake-mp4", "video/mp4")
assert seen["files"]["input_reference"][1] == b"fake-png"
assert "video_path" not in seen["data"]
assert "input_reference" not in seen["data"]
def test_generate_video_multipart_fields_are_json_encoded():
client = SGLDiffusionServerAPI(base_url="http://127.0.0.1:1234")
with tempfile.TemporaryDirectory() as tmp:
image = os.path.join(tmp, "ref.png")
with open(image, "wb") as fh:
fh.write(b"fake-png")
seen = _capture_video_request(
client,
input_reference=image,
extra_fields={"target": {"type": "image"}, "flag": False},
)
# the server json-decodes form values, so Python repr would break them
assert seen["data"]["target"] == '{"type": "image"}'
assert seen["data"]["flag"] == "false"
assert seen["data"]["prompt"] == "a cat"
def test_generate_video_sends_remote_source_video_as_video_path():
client = SGLDiffusionServerAPI(base_url="http://127.0.0.1:1234")
for source in ("https://example.com/clip.mp4", "/srv/sglang/inputs/clip.mp4"):
seen = _capture_video_request(client, video_path=source)
# A URL or a path that only exists on the server stays a string.
assert seen["files"] is None
assert seen["json"]["video_path"] == source
def test_generate_video_downloads_result_when_server_path_is_not_local():
client = SGLDiffusionServerAPI(base_url="http://127.0.0.1:1234")
def fake_post(url, **kwargs):
return _Response({"id": "job-1"})
def fake_get(url, headers=None, timeout=None, stream=False, **kwargs):
if url.endswith("/content"):
return _Response({}, status_code=200)
return _Response(
{
"id": "job-1",
"status": "completed",
# A path on the *server's* filesystem, not this one.
"file_path": "/srv/sglang/outputs/job-1.mp4",
}
)
with tempfile.TemporaryDirectory() as out_dir:
with (
mock.patch(f"{PKG}.core.server_api.requests.post", side_effect=fake_post),
mock.patch(f"{PKG}.core.server_api.requests.get", side_effect=fake_get),
):
result = client.generate_video(prompt="a cat", output_path=out_dir)
assert os.path.exists(result["file_path"])
assert os.path.realpath(
os.path.dirname(result["file_path"])
) == os.path.realpath(out_dir)
@pytest.mark.parametrize("video_id", ["../escape", "a/b", "job.1", ""])
def test_localize_video_result_rejects_unsafe_video_id(video_id):
client = SGLDiffusionServerAPI(base_url="http://127.0.0.1:1234")
status = {"status": "completed", "file_path": "/srv/out/x.mp4"}
with tempfile.TemporaryDirectory() as out_dir:
with mock.patch(f"{PKG}.core.server_api.requests.get") as get:
with pytest.raises(RuntimeError, match="unsafe id"):
client._localize_video_result(status, video_id, out_dir)
get.assert_not_called()
assert os.listdir(out_dir) == []
def test_localize_video_result_whitelists_extension_and_cleans_up_on_error():
client = SGLDiffusionServerAPI(base_url="http://127.0.0.1:1234")
class _Broken(_Response):
def iter_content(self, chunk_size=None):
yield b"partial"
raise SERVER_API.requests.exceptions.ConnectionError("dropped")
status = {"status": "completed", "file_path": "/srv/out/job-1.sh"}
with tempfile.TemporaryDirectory() as out_dir:
with mock.patch(
f"{PKG}.core.server_api.requests.get", return_value=_Response({})
):
result = client._localize_video_result(status, "job-1", out_dir)
assert result["file_path"] == os.path.join(
os.path.realpath(out_dir), "job-1.mp4"
)
os.remove(result["file_path"])
with mock.patch(
f"{PKG}.core.server_api.requests.get", return_value=_Broken({})
):
with pytest.raises(SERVER_API.requests.exceptions.ConnectionError):
client._localize_video_result(status, "job-1", out_dir)
assert os.listdir(out_dir) == []
class _HTTPErrorResponse(_Response):
def raise_for_status(self):
raise SERVER_API.requests.exceptions.HTTPError(
f"{self.status_code} Client Error", response=self
)
def _completed_video_client(job, max_polls=3):
"""Client whose job is already completed; /content answers 400, as the
server does for a job uploaded to cloud storage. Polling past
`max_polls` fails the test instead of looping until max_wait_time."""
client = SGLDiffusionServerAPI(base_url="http://127.0.0.1:1234")
calls = {"status": 0, "content": 0}
def fake_get(url, **kwargs):
if url.endswith("/content"):
calls["content"] += 1
return _HTTPErrorResponse({"detail": "use the cloud URL"}, 400)
calls["status"] += 1
assert calls["status"] <= max_polls, "kept polling a completed job"
return _Response(job)
def fake_post(url, **kwargs):
return _Response({"id": "job-1"})
patches = (
mock.patch.object(SERVER_API.requests, "get", fake_get),
mock.patch.object(SERVER_API.requests, "post", fake_post),
mock.patch.object(SERVER_API.time, "sleep"),
)
return client, calls, patches
def test_generate_video_returns_cloud_url_without_downloading_content():
url = "https://bucket.example/job-1.mp4"
client, calls, patches = _completed_video_client(
{"id": "job-1", "status": "completed", "url": url, "file_path": None}
)
with tempfile.TemporaryDirectory() as out_dir:
with patches[0], patches[1], patches[2]:
result = client.generate_video(prompt="a cat", output_path=out_dir)
# resolve_video_path fetches the url, as on main.
assert result["url"] == url
assert result["file_path"] is None
assert calls["content"] == 0
def test_generate_video_fails_fast_when_result_download_fails():
client, calls, patches = _completed_video_client(
{"id": "job-1", "status": "completed", "file_path": "/srv/out/job-1.mp4"}
)
with tempfile.TemporaryDirectory() as out_dir:
with patches[0], patches[1], patches[2]:
with pytest.raises(RuntimeError, match="400"):
client.generate_video(prompt="a cat", output_path=out_dir)
assert calls == {"status": 1, "content": 1}
def test_h3_image_conditions_use_data_uri_not_local_file_path():
node = NODES.SGLDiffusionGenerateH3()
image = torch.zeros(1, 8, 8, 3)
uri = node._image_material_uri(image)
assert uri.startswith("data:image/png;base64,")
def test_h3_rejects_local_video_reference_against_remote_server():
with pytest.raises(ValueError, match="no way to read or upload"):
NODES.SGLDiffusionGenerateH3._remote_material_uri(
"/data/clip.mp4", "reference_video", "http://remote-host.example:3000/v1"
)
def test_h3_allows_local_video_reference_against_local_server():
uri = NODES.SGLDiffusionGenerateH3._remote_material_uri(
"/data/clip.mp4", "reference_video", "http://127.0.0.1:3000/v1"
)
assert uri == "file:///data/clip.mp4"
def test_is_local_server_accepts_own_hostname_and_loopback():
is_local = NODES.SGLDiffusionGenerateH3._is_local_server
host = NODES.socket.gethostname()
assert is_local(f"http://{host}:3000/v1")
assert is_local("http://0.0.0.0:3000/v1")
assert is_local("http://127.0.0.2:3000/v1")
assert is_local("http://[::1]:3000/v1")
with mock.patch.object(
NODES.socket, "getaddrinfo", return_value=[(0, 0, 0, "", ("10.1.2.3", 0))]
):
assert not is_local("http://gpu-box:3000/v1")
def test_h3_passes_file_uri_through_for_any_server():
# a file:// URI names a path on the server, so it is never rewritten
for base_url in ("http://127.0.0.1:3000/v1", "http://remote-host.example:3000/v1"):
uri = NODES.SGLDiffusionGenerateH3._remote_material_uri(
"file:///srv/clip.mp4", "reference_video", base_url
)
assert uri == "file:///srv/clip.mp4"
# --- server-mode video jobs: no progress bar, Cancel was ignored ------------
def _video_client_with_statuses(statuses):
client = SGLDiffusionServerAPI(base_url="http://127.0.0.1:1234")
polled = iter(statuses)
def fake_get(url, **kwargs):
return _Response(next(polled))
def fake_post(url, **kwargs):
return _Response({"id": "vid1"})
return client, fake_get, fake_post
def test_generate_video_reports_server_progress():
client, fake_get, fake_post = _video_client_with_statuses(
[
{"status": "queued", "progress": 0},
{"status": "in_progress", "progress": 40},
{"status": "completed", "progress": 100, "file_path": "/tmp/x.mp4"},
]
)
seen = []
with (
mock.patch.object(SERVER_API.requests, "get", fake_get),
mock.patch.object(SERVER_API.requests, "post", fake_post),
mock.patch.object(SERVER_API.time, "sleep"),
mock.patch.object(client, "_localize_video_result", lambda s, *a: s),
):
client.generate_video(prompt="a cat", progress_callback=seen.append)
assert seen == [0, 40, 100]
def test_generate_video_stops_polling_when_interrupted():
client, fake_get, fake_post = _video_client_with_statuses(
[{"status": "in_progress", "progress": 10}] * 50
)
checks = {"n": 0}
class Interrupted(Exception):
pass
def check_interrupt():
checks["n"] += 1
if checks["n"] > 2:
raise Interrupted()
with (
mock.patch.object(SERVER_API.requests, "get", fake_get),
mock.patch.object(SERVER_API.requests, "post", fake_post),
mock.patch.object(SERVER_API.time, "sleep"),
):
with pytest.raises(Interrupted):
client.generate_video(prompt="a cat", check_interrupt=check_interrupt)
assert checks["n"] == 3 # stopped promptly, did not poll to completion
def test_video_node_wires_comfy_progress_bar_and_interrupt():
client = SGLDiffusionServerAPI(base_url="http://127.0.0.1:1234")
node = NODES.SGLDiffusionGenerateVideo()
captured = {}
def fake_generate_video(**kwargs):
captured.update(kwargs)
return {"file_path": "/tmp/x.mp4", "size": "1280x720"}
with mock.patch.object(client, "generate_video", fake_generate_video):
node.generate_video(
sgld_client=client, positive_prompt="a cat", width=1280, height=720
)
captured["progress_callback"](55)
assert sys.modules["comfy.utils"].ProgressBar.instances[-1].values == [55]
sys.modules["comfy.model_management"]._test_interrupt["flag"] = True
try:
with pytest.raises(
sys.modules["comfy.model_management"].InterruptProcessingException
):
captured["check_interrupt"]()
finally:
sys.modules["comfy.model_management"]._test_interrupt["flag"] = False
@pytest.mark.parametrize("enhance", [True, False])
def test_image_node_enhance_prompt_returns_revised_prompt(enhance):
client = SGLDiffusionServerAPI(base_url="http://127.0.0.1:1234")
png = io.BytesIO()
Image.new("RGB", (8, 8)).save(png, format="PNG")
b64 = base64.b64encode(png.getvalue()).decode()
revised = "a fluffy cat" if enhance else None
captured = {}
def fake_post(url, json=None, **kwargs):
captured.update(json)
return _Response({"data": [{"b64_json": b64, "revised_prompt": revised}]})
with mock.patch(f"{PKG}.core.server_api.requests.post", side_effect=fake_post):
image, revised_prompt = NODES.SGLDiffusionGenerateImage().generate_image(
sgld_client=client, positive_prompt="a cat", enhance_prompt=enhance
)
assert captured.get("enhance_prompt", False) == enhance
assert image.shape == (1, 8, 8, 3)
assert revised_prompt == ("a fluffy cat" if enhance else "a cat")
def test_video_node_sends_source_video_and_enhance_prompt():
client = SGLDiffusionServerAPI(base_url="http://127.0.0.1:1234")
captured = {}
class SourceVideo:
def save_to(self, path):
with open(path, "wb") as fh:
fh.write(b"source-mp4")
def fake_generate_video(**kwargs):
captured.update(kwargs)
with open(kwargs["video_path"], "rb") as fh:
captured["video_bytes"] = fh.read()
return {"file_path": "/tmp/x.mp4", "revised_prompt": "a fluffy cat"}
with mock.patch.object(client, "generate_video", fake_generate_video):
_, video_path, revised_prompt = (
NODES.SGLDiffusionGenerateVideo().generate_video(
sgld_client=client,
positive_prompt="a cat",
video=SourceVideo(),
enhance_prompt=True,
)
)
os.remove(captured["video_path"])
assert captured["video_bytes"] == b"source-mp4"
assert captured["extra_fields"] == {"enhance_prompt": True}
assert (video_path, revised_prompt) == ("/tmp/x.mp4", "a fluffy cat")
@@ -1,5 +1,6 @@
import base64
import io
import logging
import os
import shutil
import subprocess
@@ -13,6 +14,8 @@ import torch
from comfy_api.input import VideoInput
from PIL import Image
logger = logging.getLogger(__name__)
def _ensure_dir(path: str) -> None:
os.makedirs(path, exist_ok=True)
@@ -82,7 +85,16 @@ def is_empty_image(image: torch.Tensor, tolerance: float = 1e-6) -> bool:
def get_image_path(image: torch.Tensor) -> str:
"""
Save tensor image to ComfyUI temp directory as PNG and return the path.
This input only has a single image slot, so a batch keeps its first
image (as before) and logs a warning that the rest are dropped.
"""
if image.dim() == 4 and image.shape[0] > 1:
logger.warning(
"get_image_path received a batch of %d images but this input only "
"accepts one; using the first. Use 'Image From Batch' to pick another.",
image.shape[0],
)
temp_dir = folder_paths.get_temp_directory()
# Build file name
@@ -131,6 +143,14 @@ def convert_b64_to_tensor_image(b64_image: str) -> torch.Tensor:
class SGLDVideoInput(VideoInput):
"""Minimal fallback used only when ComfyUI's own file-backed VideoInput
isn't importable. It cannot decode frames/audio or re-encode on save, so
it fails loudly on those rather than silently returning wrong data (the
previous ``get_components`` returned ``[video_path]`` instead of a
``VideoComponents``, and ``save_to`` dropped ``format``/``codec``/
``metadata`` without telling the caller).
"""
def __init__(self, video_path: str, height: int, width: int):
super().__init__()
@@ -148,16 +168,19 @@ class SGLDVideoInput(VideoInput):
return self.width, self.height
def get_components(self):
"""
Returns the components of the video input.
This is required by the VideoInput abstract base class.
"""
return [self.video_path]
raise NotImplementedError(
"SGLDVideoInput cannot decode frames/audio (no ComfyUI "
"VideoFromFile available in this environment); pass the file "
"at video_path to a node that reads files directly instead."
)
def save_to(self, path: str, format=None, codec=None, metadata=None):
"""
Abstract method to save the video input to a file.
"""
if format is not None or codec is not None or metadata is not None:
raise NotImplementedError(
"SGLDVideoInput can only copy the file as-is; format/codec/"
"metadata conversion needs ComfyUI's own VideoFromFile, "
"which is unavailable in this environment."
)
if not os.path.exists(self.video_path):
raise FileNotFoundError(
f"Video not found at '{self.video_path}'. This file must be "
@@ -236,7 +259,18 @@ def convert_video_to_comfy_video(
video_path: str, height: int, width: int
) -> VideoInput:
"""
Convert video to ComfyUI VIDEO format (VideoInput).
Convert a video file on disk to ComfyUI's VIDEO type.
Prefers ComfyUI's own file-backed ``VideoFromFile``: it probes the real
file for dimensions/frames/audio, so it is also immune to the
width/height we were told (which can be wrong, e.g. when the server
picked a different size than requested to match an image-to-video
input). ``height``/``width`` are only used by the ``SGLDVideoInput``
fallback below.
"""
video_input = SGLDVideoInput(video_path, height, width)
return video_input
try:
from comfy_api.input_impl import VideoFromFile
return VideoFromFile(video_path)
except ImportError:
return SGLDVideoInput(video_path, height, width)
@@ -33,7 +33,13 @@ def _load_nodes():
comfy_api_input = types.ModuleType("comfy_api.input")
comfy_api_input.VideoInput = type("VideoInput", (), {})
comfy_api.input = comfy_api_input
comfy = types.ModuleType("comfy")
comfy.model_management = types.ModuleType("comfy.model_management")
comfy.utils = types.ModuleType("comfy.utils")
stubs = {
"comfy": comfy,
"comfy.model_management": comfy.model_management,
"comfy.utils": comfy.utils,
"folder_paths": folder_paths,
"comfy_api": comfy_api,
"comfy_api.input": comfy_api_input,