[Feat] comfyui: video-to-video input and prompt enhancement for server-mode nodes (#43673)
This commit is contained in:
@@ -34,7 +34,8 @@ Connect to a standalone SGLang Diffusion server.
|
||||
2. **Connect to Server**: Use the `SGLDiffusion Server Model` node to connect (default: `http://localhost:3000/v1`).
|
||||
3. **Generate Content**:
|
||||
- `SGLDiffusion Generate Image`: For text-to-image and image editing. Set `num_images` (1 to 10) to get several images from one request as one `IMAGE` batch, in order. Image `i` uses `seed + i`, so the same settings and seed reproduce the same batch; a seed of `-1` lets the server pick a random seed.
|
||||
- `SGLDiffusion Generate Video`: For text-to-video and image-to-video.
|
||||
- `SGLDiffusion Generate Video`: For text-to-video, image-to-video, and video-to-video (connect a `VIDEO`).
|
||||
- Both nodes have an `enhance_prompt` toggle (the server needs `--prompt-enhancer-config`); the rewritten prompt comes out as `revised_prompt`.
|
||||
- `SGLDiffusion Generate MiniMax-H3`: For joint video-and-audio on a standalone server.
|
||||
4. **LoRA Support**: Use `SGLDiffusion Server Set LoRA` and `SGLDiffusion Server Unset LoRA`.
|
||||
|
||||
|
||||
@@ -3,24 +3,43 @@ SGLang Diffusion Server API client.
|
||||
Provides a low-level interface for interacting with SGLang Diffusion HTTP server.
|
||||
"""
|
||||
|
||||
import contextlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
from typing import Any, Dict, Optional
|
||||
from typing import Any, Callable, Dict, Optional
|
||||
|
||||
import requests
|
||||
|
||||
# Server-supplied names used to build a local path for a downloaded video.
|
||||
_SAFE_VIDEO_ID = re.compile(r"[A-Za-z0-9_-]+")
|
||||
_VIDEO_EXTENSIONS = (".mp4", ".webm", ".mov", ".mkv")
|
||||
_MAX_VIDEO_BYTES = 2 * 1024**3
|
||||
_DOWNLOAD_DEADLINE_S = 600
|
||||
|
||||
|
||||
class SGLDiffusionServerAPI:
|
||||
"""Client for SGLang Diffusion HTTP server API."""
|
||||
|
||||
def __init__(self, base_url: str, api_key: str = "sk-proj-1234567890"):
|
||||
def __init__(
|
||||
self,
|
||||
base_url: str,
|
||||
api_key: str = "sk-proj-1234567890",
|
||||
image_timeout: int = 300,
|
||||
):
|
||||
"""
|
||||
Initialize the API client.
|
||||
|
||||
Args:
|
||||
base_url: Base URL of the SGLang Diffusion server (e.g., "http://localhost:30010/v1")
|
||||
api_key: API key for authentication (default: "sk-proj-1234567890")
|
||||
image_timeout: Seconds to wait for an image generate/edit request
|
||||
before timing out. A large job (big size, or a first request
|
||||
that triggers torch.compile/warmup) can need more than the
|
||||
previous hardcoded 300s.
|
||||
"""
|
||||
self.image_timeout = image_timeout
|
||||
# Ensure base_url doesn't end with /v1 if it's already there
|
||||
if base_url.endswith("/v1"):
|
||||
self.base_url = base_url
|
||||
@@ -80,6 +99,7 @@ class SGLDiffusionServerAPI:
|
||||
background: Optional[str] = "auto",
|
||||
output_format: Optional[str] = None,
|
||||
generator_device: Optional[str] = "cuda",
|
||||
enhance_prompt: bool = False,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Generate or edit an image using SGLang Diffusion API.
|
||||
@@ -105,6 +125,7 @@ class SGLDiffusionServerAPI:
|
||||
background: Background type ("auto", "transparent", "opaque")
|
||||
output_format: Output format ("png", "jpeg", "webp")
|
||||
generator_device: Device for random generator ("cuda" or "cpu")
|
||||
enhance_prompt: Rewrite the prompt on the server first
|
||||
|
||||
Returns:
|
||||
Dictionary containing the API response with generated/edited image data
|
||||
@@ -133,6 +154,7 @@ class SGLDiffusionServerAPI:
|
||||
background=background,
|
||||
output_format=output_format,
|
||||
generator_device=generator_device,
|
||||
enhance_prompt=enhance_prompt,
|
||||
)
|
||||
|
||||
# If image_path is provided, use edit endpoint
|
||||
@@ -172,7 +194,7 @@ class SGLDiffusionServerAPI:
|
||||
files=files,
|
||||
data=data,
|
||||
headers=headers,
|
||||
timeout=300, # 5 minutes timeout for generation
|
||||
timeout=self.image_timeout,
|
||||
)
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
@@ -196,7 +218,7 @@ class SGLDiffusionServerAPI:
|
||||
f"{self.base_url}/images/generations",
|
||||
json=payload,
|
||||
headers=self.headers,
|
||||
timeout=300, # 5 minutes timeout for generation
|
||||
timeout=self.image_timeout,
|
||||
)
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
@@ -219,12 +241,19 @@ class SGLDiffusionServerAPI:
|
||||
enable_teacache: bool = False,
|
||||
generator_device: Optional[str] = "cuda",
|
||||
input_reference: Optional[str] = None,
|
||||
video_path: Optional[str] = None,
|
||||
output_path: Optional[str] = None,
|
||||
extra_fields: Optional[Dict[str, Any]] = None,
|
||||
progress_callback: Optional[Callable[[int], None]] = None,
|
||||
check_interrupt: Optional[Callable[[], None]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Generate a video using SGLang Diffusion API and wait for completion.
|
||||
|
||||
``progress_callback`` gets the job's ``progress`` (0-100) on each poll.
|
||||
``check_interrupt`` is called between polls and may raise to stop
|
||||
waiting; the server has no abort yet, so the job itself keeps running.
|
||||
|
||||
Args:
|
||||
prompt: Text prompt for video generation
|
||||
size: Video size in format "WIDTHxHEIGHT" (e.g., "1280x720")
|
||||
@@ -240,6 +269,8 @@ class SGLDiffusionServerAPI:
|
||||
enable_teacache: Enable TEA cache acceleration
|
||||
generator_device: Device for random generator ("cuda" or "cpu")
|
||||
input_reference: Path to input reference image for image-to-video
|
||||
video_path: Source clip for video-to-video. A local file is
|
||||
uploaded; a URL or server-side path is sent as is
|
||||
extra_fields: Model-specific request fields merged into the payload
|
||||
last, so a caller can reach a model's own request surface
|
||||
(MiniMax-H3's `task`/`conditions`/`target`, per-model flow
|
||||
@@ -258,48 +289,89 @@ class SGLDiffusionServerAPI:
|
||||
else:
|
||||
size = "720x1280"
|
||||
|
||||
# Prepare request payload
|
||||
payload: Dict[str, Any] = {
|
||||
# Prepare request fields shared by the JSON and multipart paths.
|
||||
# `output_path` is deliberately never sent: it used to be the
|
||||
# ComfyUI node's own temp directory, which only exists on this
|
||||
# machine and breaks as soon as the server runs elsewhere. Letting
|
||||
# the server pick its own output location and fetching the result
|
||||
# (see below) works for both the local and remote case.
|
||||
fields: Dict[str, Any] = {
|
||||
"prompt": prompt,
|
||||
"size": size,
|
||||
}
|
||||
|
||||
# Add optional parameters
|
||||
if seconds is not None:
|
||||
payload["seconds"] = seconds
|
||||
fields["seconds"] = seconds
|
||||
if fps is not None:
|
||||
payload["fps"] = fps
|
||||
fields["fps"] = fps
|
||||
if num_frames is not None:
|
||||
payload["num_frames"] = num_frames
|
||||
fields["num_frames"] = num_frames
|
||||
if negative_prompt:
|
||||
payload["negative_prompt"] = negative_prompt
|
||||
fields["negative_prompt"] = negative_prompt
|
||||
if guidance_scale is not None:
|
||||
payload["guidance_scale"] = guidance_scale
|
||||
fields["guidance_scale"] = guidance_scale
|
||||
if num_inference_steps is not None:
|
||||
payload["num_inference_steps"] = num_inference_steps
|
||||
fields["num_inference_steps"] = num_inference_steps
|
||||
if seed is not None and seed >= 0:
|
||||
payload["seed"] = seed
|
||||
fields["seed"] = seed
|
||||
if enable_teacache:
|
||||
payload["enable_teacache"] = True
|
||||
fields["enable_teacache"] = True
|
||||
if generator_device:
|
||||
payload["generator_device"] = generator_device
|
||||
if input_reference:
|
||||
payload["input_reference"] = input_reference
|
||||
if output_path:
|
||||
payload["output_path"] = output_path
|
||||
fields["generator_device"] = generator_device
|
||||
# merged last so a model-specific field wins over a generic default of
|
||||
# the same name (H3 sizes its output from `target`, not `size`)
|
||||
if extra_fields:
|
||||
payload.update(extra_fields)
|
||||
fields.update(extra_fields)
|
||||
|
||||
# Upload local files instead of sending their paths: a server on
|
||||
# another machine cannot read this machine's filesystem (the images
|
||||
# endpoint already does this for image_path). The server takes an
|
||||
# uploaded source clip as `video_reference`.
|
||||
string_fields: Dict[str, Any] = {}
|
||||
uploads: Dict[str, str] = {}
|
||||
for form_key, json_key, path in (
|
||||
("input_reference", "input_reference", input_reference),
|
||||
("video_reference", "video_path", video_path),
|
||||
):
|
||||
if not path:
|
||||
continue
|
||||
if self._is_local_file(path):
|
||||
uploads[form_key] = path
|
||||
else:
|
||||
string_fields[json_key] = path
|
||||
|
||||
try:
|
||||
# Create video generation job
|
||||
response = requests.post(
|
||||
f"{self.base_url}/videos",
|
||||
json=payload,
|
||||
headers=self.headers,
|
||||
timeout=30,
|
||||
)
|
||||
if uploads:
|
||||
# JSON, not repr: the server json-decodes form values, so a
|
||||
# nested dict or False must not arrive as Python's repr
|
||||
data = {
|
||||
key: (
|
||||
value if isinstance(value, (str, bytes)) else json.dumps(value)
|
||||
)
|
||||
for key, value in {**fields, **string_fields}.items()
|
||||
}
|
||||
with contextlib.ExitStack() as stack:
|
||||
files = {
|
||||
form_key: (
|
||||
os.path.basename(path),
|
||||
stack.enter_context(open(path, "rb")),
|
||||
self._get_content_type(path),
|
||||
)
|
||||
for form_key, path in uploads.items()
|
||||
}
|
||||
response = requests.post(
|
||||
f"{self.base_url}/videos",
|
||||
files=files,
|
||||
data=data,
|
||||
headers={"Authorization": f"Bearer {self.api_key}"},
|
||||
timeout=30,
|
||||
)
|
||||
else:
|
||||
response = requests.post(
|
||||
f"{self.base_url}/videos",
|
||||
json={**fields, **string_fields},
|
||||
headers=self.headers,
|
||||
timeout=30,
|
||||
)
|
||||
response.raise_for_status()
|
||||
video_job = response.json()
|
||||
video_id = video_job.get("id")
|
||||
@@ -312,6 +384,8 @@ class SGLDiffusionServerAPI:
|
||||
start_time = time.time()
|
||||
|
||||
while time.time() - start_time < max_wait_time:
|
||||
if check_interrupt:
|
||||
check_interrupt()
|
||||
try:
|
||||
status_response = requests.get(
|
||||
f"{self.base_url}/videos/{video_id}",
|
||||
@@ -324,8 +398,13 @@ class SGLDiffusionServerAPI:
|
||||
# Reset error counter on successful request
|
||||
consecutive_errors = 0
|
||||
|
||||
if progress_callback and status.get("progress") is not None:
|
||||
progress_callback(int(status["progress"]))
|
||||
|
||||
if status.get("status") == "completed":
|
||||
return status
|
||||
# Localize outside this try, so a failed download
|
||||
# fails fast instead of being retried as a poll error.
|
||||
break
|
||||
elif status.get("status") == "failed":
|
||||
error = status.get("error", {})
|
||||
error_msg = (
|
||||
@@ -350,14 +429,80 @@ class SGLDiffusionServerAPI:
|
||||
f"Network error after {consecutive_errors} consecutive failures: {str(e)}"
|
||||
)
|
||||
|
||||
time.sleep(poll_interval)
|
||||
|
||||
raise TimeoutError(
|
||||
f"Video generation timed out after {max_wait_time} seconds"
|
||||
)
|
||||
# Sleep in short slices so Cancel is honoured within ~1s.
|
||||
for _ in range(poll_interval):
|
||||
if check_interrupt:
|
||||
check_interrupt()
|
||||
time.sleep(1)
|
||||
else:
|
||||
raise TimeoutError(
|
||||
f"Video generation timed out after {max_wait_time} seconds"
|
||||
)
|
||||
return self._localize_video_result(status, video_id, output_path)
|
||||
except requests.exceptions.RequestException as e:
|
||||
raise RuntimeError(f"Failed to generate video: {str(e)}")
|
||||
|
||||
def _localize_video_result(
|
||||
self, status: Dict[str, Any], video_id: str, output_path: Optional[str]
|
||||
) -> Dict[str, Any]:
|
||||
"""Make sure ``file_path`` in a completed job points at a local file.
|
||||
|
||||
The server's own ``file_path`` is only readable when the server runs
|
||||
on this machine. When it isn't (no such local file), download the
|
||||
result through ``GET /videos/{id}/content`` instead of trusting a
|
||||
path that belongs to a different filesystem. A job uploaded to cloud
|
||||
storage has no file_path and a ``url`` instead; ``/content`` refuses
|
||||
those, so it is returned as is and the node fetches the url.
|
||||
"""
|
||||
file_path = status.get("file_path")
|
||||
if file_path and os.path.exists(file_path):
|
||||
return status
|
||||
if not file_path and status.get("url"):
|
||||
return status
|
||||
if not isinstance(video_id, str) or not _SAFE_VIDEO_ID.fullmatch(video_id):
|
||||
raise RuntimeError(f"Refusing to save video with unsafe id {video_id!r}")
|
||||
ext = os.path.splitext(file_path or "")[1].lower() or ".mp4"
|
||||
if ext not in _VIDEO_EXTENSIONS:
|
||||
ext = ".mp4"
|
||||
target_dir = os.path.realpath(output_path or os.getcwd())
|
||||
os.makedirs(target_dir, exist_ok=True)
|
||||
local_path = os.path.realpath(os.path.join(target_dir, f"{video_id}{ext}"))
|
||||
if os.path.commonpath([target_dir, local_path]) != target_dir:
|
||||
raise RuntimeError(f"Refusing to save video outside {target_dir}")
|
||||
deadline = time.monotonic() + _DOWNLOAD_DEADLINE_S
|
||||
response = requests.get(
|
||||
f"{self.base_url}/videos/{video_id}/content",
|
||||
headers=self.headers,
|
||||
timeout=300,
|
||||
stream=True,
|
||||
)
|
||||
try:
|
||||
response.raise_for_status()
|
||||
written = 0
|
||||
with open(local_path, "wb") as fh:
|
||||
for chunk in response.iter_content(chunk_size=1024 * 1024):
|
||||
written += len(chunk)
|
||||
if written > _MAX_VIDEO_BYTES:
|
||||
raise RuntimeError(
|
||||
f"Video download exceeded {_MAX_VIDEO_BYTES} bytes"
|
||||
)
|
||||
if time.monotonic() > deadline:
|
||||
raise RuntimeError(
|
||||
f"Video download exceeded {_DOWNLOAD_DEADLINE_S}s"
|
||||
)
|
||||
fh.write(chunk)
|
||||
except BaseException:
|
||||
with contextlib.suppress(OSError):
|
||||
os.remove(local_path)
|
||||
raise
|
||||
finally:
|
||||
close = getattr(response, "close", None)
|
||||
if close is not None:
|
||||
close()
|
||||
status = dict(status)
|
||||
status["file_path"] = local_path
|
||||
return status
|
||||
|
||||
def _build_image_common_params(
|
||||
self,
|
||||
prompt: str,
|
||||
@@ -372,6 +517,7 @@ class SGLDiffusionServerAPI:
|
||||
background: Optional[str] = None,
|
||||
output_format: Optional[str] = None,
|
||||
generator_device: Optional[str] = None,
|
||||
enhance_prompt: bool = False,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Build common parameters for both image generation and editing.
|
||||
@@ -403,9 +549,17 @@ class SGLDiffusionServerAPI:
|
||||
params["output_format"] = output_format
|
||||
if generator_device:
|
||||
params["generator_device"] = generator_device
|
||||
if enhance_prompt:
|
||||
params["enhance_prompt"] = True
|
||||
|
||||
return params
|
||||
|
||||
@staticmethod
|
||||
def _is_local_file(path: str) -> bool:
|
||||
return not path.startswith(("http://", "https://", "data:")) and os.path.isfile(
|
||||
path
|
||||
)
|
||||
|
||||
def _get_content_type(self, file_path: str) -> str:
|
||||
"""Get content type based on file extension."""
|
||||
ext = os.path.splitext(file_path)[1].lower()
|
||||
@@ -414,6 +568,10 @@ class SGLDiffusionServerAPI:
|
||||
".jpg": "image/jpeg",
|
||||
".jpeg": "image/jpeg",
|
||||
".webp": "image/webp",
|
||||
".mp4": "video/mp4",
|
||||
".webm": "video/webm",
|
||||
".mov": "video/quicktime",
|
||||
".mkv": "video/x-matroska",
|
||||
}
|
||||
return content_types.get(ext, "image/png")
|
||||
|
||||
@@ -422,6 +580,7 @@ class SGLDiffusionServerAPI:
|
||||
lora_nickname: str,
|
||||
lora_path: Optional[str] = None,
|
||||
target: str = "all",
|
||||
strength: Optional[float] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Set a LoRA adapter for the specified transformer(s).
|
||||
@@ -435,6 +594,8 @@ class SGLDiffusionServerAPI:
|
||||
- "transformer": Apply only to the primary transformer (high noise for Wan2.2)
|
||||
- "transformer_2": Apply only to transformer_2 (low noise for Wan2.2)
|
||||
- "critic": Apply only to the critic model
|
||||
strength: LoRA merge strength. The server defaults to 1.0 when
|
||||
not given; values < 1.0 reduce the effect.
|
||||
|
||||
Returns:
|
||||
Dictionary containing the API response with status and message
|
||||
@@ -451,6 +612,8 @@ class SGLDiffusionServerAPI:
|
||||
# Add optional lora_path if provided
|
||||
if lora_path:
|
||||
payload["lora_path"] = lora_path
|
||||
if strength is not None:
|
||||
payload["strength"] = strength
|
||||
|
||||
try:
|
||||
response = requests.post(
|
||||
|
||||
@@ -3,9 +3,15 @@ ComfyUI nodes for SGLang Diffusion integration.
|
||||
Provides nodes for connecting to SGLang Diffusion server and generating images/videos.
|
||||
"""
|
||||
|
||||
import base64
|
||||
import ipaddress
|
||||
import os
|
||||
import socket
|
||||
import urllib.parse
|
||||
import uuid
|
||||
|
||||
import comfy.model_management
|
||||
import comfy.utils
|
||||
import folder_paths
|
||||
import torch
|
||||
|
||||
@@ -74,6 +80,15 @@ class SGLDH3DistilledSigmas:
|
||||
return (torch.tensor(distilled_h3_sigmas(model_type)),)
|
||||
|
||||
|
||||
def _comfy_job_hooks() -> dict:
|
||||
"""Wire a server-mode job into ComfyUI's progress bar and Cancel button."""
|
||||
pbar = comfy.utils.ProgressBar(100)
|
||||
return {
|
||||
"progress_callback": pbar.update_absolute,
|
||||
"check_interrupt": comfy.model_management.throw_exception_if_processing_interrupted,
|
||||
}
|
||||
|
||||
|
||||
class SGLDLoraLoader:
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
@@ -216,7 +231,21 @@ class SGLDiffusionServerModel:
|
||||
"multiline": False,
|
||||
},
|
||||
),
|
||||
}
|
||||
},
|
||||
"optional": {
|
||||
"image_timeout": (
|
||||
"INT",
|
||||
{
|
||||
"default": 300,
|
||||
"min": 1,
|
||||
"max": 3600,
|
||||
"tooltip": "Seconds to wait for an image "
|
||||
"generate/edit request (e.g. a large job, or "
|
||||
"torch.compile warmup on the first request, can "
|
||||
"need more than 300s)",
|
||||
},
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("SGLD_CLIENT", "STRING")
|
||||
@@ -224,9 +253,11 @@ class SGLDiffusionServerModel:
|
||||
FUNCTION = "load_server"
|
||||
CATEGORY = "SGLDiffusion"
|
||||
|
||||
def load_server(self, base_url: str, api_key: str):
|
||||
def load_server(self, base_url: str, api_key: str, image_timeout: int = 300):
|
||||
"""Initialize OpenAI client for SGLang Diffusion server."""
|
||||
client = SGLDiffusionServerAPI(base_url=base_url, api_key=api_key)
|
||||
client = SGLDiffusionServerAPI(
|
||||
base_url=base_url, api_key=api_key, image_timeout=image_timeout
|
||||
)
|
||||
try:
|
||||
model_info = client.get_model_info()
|
||||
# Format model_info as a readable string
|
||||
@@ -330,11 +361,19 @@ class SGLDiffusionGenerateImage:
|
||||
"one IMAGE batch. Image i uses seed + i.",
|
||||
},
|
||||
),
|
||||
"enhance_prompt": (
|
||||
"BOOLEAN",
|
||||
{
|
||||
"default": False,
|
||||
"tooltip": "Rewrite the prompt on the server (needs "
|
||||
"--prompt-enhancer-config); returned as revised_prompt",
|
||||
},
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("IMAGE",)
|
||||
RETURN_NAMES = ("image",)
|
||||
RETURN_TYPES = ("IMAGE", "STRING")
|
||||
RETURN_NAMES = ("image", "revised_prompt")
|
||||
FUNCTION = "generate_image"
|
||||
CATEGORY = "SGLDiffusion"
|
||||
OUTPUT_NODE = False
|
||||
@@ -352,6 +391,7 @@ class SGLDiffusionGenerateImage:
|
||||
height: int = 1024,
|
||||
enable_teacache: bool = False,
|
||||
num_images: int = 1,
|
||||
enhance_prompt: bool = False,
|
||||
):
|
||||
"""Generate image using SGLang Diffusion API."""
|
||||
if not positive_prompt:
|
||||
@@ -382,6 +422,8 @@ class SGLDiffusionGenerateImage:
|
||||
request_params["seed"] = seed
|
||||
if enable_teacache:
|
||||
request_params["enable_teacache"] = True
|
||||
if enhance_prompt:
|
||||
request_params["enhance_prompt"] = True
|
||||
if image is not None:
|
||||
# If the image is empty, use the size of the image to generate the image
|
||||
if is_empty_image(image):
|
||||
@@ -397,7 +439,9 @@ class SGLDiffusionGenerateImage:
|
||||
except Exception as e:
|
||||
raise RuntimeError(f"Failed to generate image: {str(e)}")
|
||||
|
||||
return (_decode_image_batch(response, num_images),)
|
||||
images = _decode_image_batch(response, num_images)
|
||||
revised_prompt = response["data"][0].get("revised_prompt") or positive_prompt
|
||||
return (images, revised_prompt)
|
||||
|
||||
|
||||
class SGLDiffusionGenerateVideo:
|
||||
@@ -431,6 +475,10 @@ class SGLDiffusionGenerateVideo:
|
||||
"tooltip": "input image to use for image-to-video",
|
||||
},
|
||||
),
|
||||
"video": (
|
||||
"VIDEO",
|
||||
{"tooltip": "source clip for video-to-video"},
|
||||
),
|
||||
"seed": (
|
||||
"INT",
|
||||
{
|
||||
@@ -508,11 +556,19 @@ class SGLDiffusionGenerateVideo:
|
||||
"default": False,
|
||||
},
|
||||
),
|
||||
"enhance_prompt": (
|
||||
"BOOLEAN",
|
||||
{
|
||||
"default": False,
|
||||
"tooltip": "Rewrite the prompt on the server (needs "
|
||||
"--prompt-enhancer-config); returned as revised_prompt",
|
||||
},
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("VIDEO", "STRING")
|
||||
RETURN_NAMES = ("video", "video_path")
|
||||
RETURN_TYPES = ("VIDEO", "STRING", "STRING")
|
||||
RETURN_NAMES = ("video", "video_path", "revised_prompt")
|
||||
FUNCTION = "generate_video"
|
||||
CATEGORY = "SGLDiffusion"
|
||||
OUTPUT_NODE = False
|
||||
@@ -532,6 +588,8 @@ class SGLDiffusionGenerateVideo:
|
||||
fps: int = 24,
|
||||
seconds: int = 5,
|
||||
enable_teacache: bool = False,
|
||||
video=None,
|
||||
enhance_prompt: bool = False,
|
||||
):
|
||||
"""Generate video using SGLang Diffusion API."""
|
||||
if not positive_prompt:
|
||||
@@ -570,18 +628,35 @@ class SGLDiffusionGenerateVideo:
|
||||
request_params["size"] = size
|
||||
else:
|
||||
request_params["input_reference"] = get_image_path(image)
|
||||
if video is not None:
|
||||
request_params["video_path"] = os.path.join(
|
||||
output_dir, f"sgl_input_{uuid.uuid4().hex[:8]}.mp4"
|
||||
)
|
||||
video.save_to(request_params["video_path"])
|
||||
if enhance_prompt:
|
||||
request_params["extra_fields"] = {"enhance_prompt": True}
|
||||
|
||||
# Call API
|
||||
try:
|
||||
response = sgld_client.generate_video(**request_params)
|
||||
response = sgld_client.generate_video(
|
||||
**request_params, **_comfy_job_hooks()
|
||||
)
|
||||
video_path = resolve_video_path(
|
||||
response.get("file_path"), response.get("url")
|
||||
)
|
||||
# The server may have resolved a different size than requested
|
||||
# (e.g. to match an image-to-video input's aspect ratio).
|
||||
resolved_size = response.get("size", "")
|
||||
if resolved_size:
|
||||
width, height = (int(v) for v in resolved_size.split("x"))
|
||||
video = convert_video_to_comfy_video(video_path, height, width)
|
||||
except comfy.model_management.InterruptProcessingException:
|
||||
raise
|
||||
except Exception as e:
|
||||
raise RuntimeError(f"Failed to generate video: {str(e)}")
|
||||
|
||||
return (video, video_path)
|
||||
revised_prompt = response.get("revised_prompt") or positive_prompt
|
||||
return (video, video_path, revised_prompt)
|
||||
|
||||
|
||||
class SGLDiffusionGenerateH3:
|
||||
@@ -668,11 +743,61 @@ class SGLDiffusionGenerateH3:
|
||||
OUTPUT_NODE = False
|
||||
|
||||
@staticmethod
|
||||
def _material_uri(value: str) -> str:
|
||||
"""Local paths become file:// URIs; remote URLs are passed through."""
|
||||
def _image_material_uri(image: torch.Tensor) -> str:
|
||||
"""Embed the image inline as a base64 data URI.
|
||||
|
||||
A `file://` path only works when the SGLD server runs on this same
|
||||
machine; the server's H3 material loader accepts `data:` URIs
|
||||
directly (no upload endpoint needed), so this works locally and
|
||||
remotely alike.
|
||||
"""
|
||||
path = get_image_path(image)
|
||||
try:
|
||||
with open(path, "rb") as fh:
|
||||
encoded = base64.b64encode(fh.read()).decode("ascii")
|
||||
finally:
|
||||
os.remove(path)
|
||||
return f"data:image/png;base64,{encoded}"
|
||||
|
||||
@staticmethod
|
||||
def _is_local_server(base_url: str) -> bool:
|
||||
host = (urllib.parse.urlparse(base_url).hostname or "").lower()
|
||||
if not host:
|
||||
return False
|
||||
if host in ("localhost", "0.0.0.0", "::"):
|
||||
return True
|
||||
own = {socket.gethostname().lower(), socket.getfqdn().lower()}
|
||||
if host in own:
|
||||
return True
|
||||
try:
|
||||
infos = socket.getaddrinfo(host, None)
|
||||
except (OSError, UnicodeError):
|
||||
return False
|
||||
addrs = {info[4][0].split("%", 1)[0] for info in infos}
|
||||
return bool(addrs) and all(
|
||||
ipaddress.ip_address(addr).is_loopback for addr in addrs
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _remote_material_uri(cls, value: str, kind: str, base_url: str) -> str:
|
||||
"""Remote URLs pass through. A local path only works when the SGLD
|
||||
server is also on this machine: H3's server-side `/v1/videos`
|
||||
endpoint has no multipart slot for the video/audio conditions this
|
||||
node sends through `extra_fields.conditions`, unlike the image
|
||||
conditions above (which sidestep that by embedding bytes inline).
|
||||
Raise clearly instead of building a `file://` URI that will
|
||||
silently fail against any server that isn't local.
|
||||
"""
|
||||
# file:// already names a server-side path
|
||||
if value.startswith(("http://", "https://", "file://")):
|
||||
return value
|
||||
return f"file://{os.path.abspath(value)}"
|
||||
if cls._is_local_server(base_url):
|
||||
return f"file://{os.path.abspath(value)}"
|
||||
raise ValueError(
|
||||
f"{kind} must be an http(s) URL when the SGLD server "
|
||||
f"({base_url}) isn't on this machine; got a local path "
|
||||
f"({value!r}), which that server has no way to read or upload."
|
||||
)
|
||||
|
||||
def generate(
|
||||
self,
|
||||
@@ -705,7 +830,7 @@ class SGLDiffusionGenerateH3:
|
||||
conditions.append(
|
||||
{
|
||||
"type": "image",
|
||||
"uri": self._material_uri(get_image_path(first_frame)),
|
||||
"uri": self._image_material_uri(first_frame),
|
||||
"role": "keyframe",
|
||||
"frame_index": 0,
|
||||
}
|
||||
@@ -714,7 +839,7 @@ class SGLDiffusionGenerateH3:
|
||||
conditions.append(
|
||||
{
|
||||
"type": "image",
|
||||
"uri": self._material_uri(get_image_path(last_frame)),
|
||||
"uri": self._image_material_uri(last_frame),
|
||||
"role": "keyframe",
|
||||
"frame_index": -1,
|
||||
}
|
||||
@@ -723,7 +848,7 @@ class SGLDiffusionGenerateH3:
|
||||
conditions.append(
|
||||
{
|
||||
"type": "image",
|
||||
"uri": self._material_uri(get_image_path(reference_image)),
|
||||
"uri": self._image_material_uri(reference_image),
|
||||
"role": "reference",
|
||||
}
|
||||
)
|
||||
@@ -731,7 +856,9 @@ class SGLDiffusionGenerateH3:
|
||||
conditions.append(
|
||||
{
|
||||
"type": "video",
|
||||
"uri": self._material_uri(reference_video),
|
||||
"uri": self._remote_material_uri(
|
||||
reference_video, "reference_video", sgld_client.base_url
|
||||
),
|
||||
"role": "reference",
|
||||
}
|
||||
)
|
||||
@@ -739,7 +866,9 @@ class SGLDiffusionGenerateH3:
|
||||
conditions.append(
|
||||
{
|
||||
"type": "audio",
|
||||
"uri": self._material_uri(reference_audio),
|
||||
"uri": self._remote_material_uri(
|
||||
reference_audio, "reference_audio", sgld_client.base_url
|
||||
),
|
||||
"role": "reference",
|
||||
}
|
||||
)
|
||||
@@ -783,10 +912,14 @@ class SGLDiffusionGenerateH3:
|
||||
request_params["seed"] = seed
|
||||
|
||||
try:
|
||||
response = sgld_client.generate_video(**request_params)
|
||||
response = sgld_client.generate_video(
|
||||
**request_params, **_comfy_job_hooks()
|
||||
)
|
||||
video_path = resolve_video_path(
|
||||
response.get("file_path"), response.get("url")
|
||||
)
|
||||
except comfy.model_management.InterruptProcessingException:
|
||||
raise
|
||||
except Exception as e:
|
||||
raise RuntimeError(f"Failed to generate MiniMax-H3 video: {str(e)}")
|
||||
|
||||
@@ -838,6 +971,17 @@ class SGLDiffusionServerSetLora:
|
||||
"tooltip": "Which transformer(s) to apply the LoRA to",
|
||||
},
|
||||
),
|
||||
"strength": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 1.0,
|
||||
"min": 0.0,
|
||||
"max": 10.0,
|
||||
"step": 0.01,
|
||||
"tooltip": "LoRA merge strength; the server's own "
|
||||
"default is 1.0",
|
||||
},
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
@@ -853,6 +997,7 @@ class SGLDiffusionServerSetLora:
|
||||
lora_name: str = "",
|
||||
lora_nickname: str = "",
|
||||
target: str = "all",
|
||||
strength: float = 1.0,
|
||||
):
|
||||
"""Set LoRA adapter using SGLang Diffusion API."""
|
||||
if lora_nickname == "":
|
||||
@@ -863,6 +1008,7 @@ class SGLDiffusionServerSetLora:
|
||||
"lora_nickname": lora_nickname,
|
||||
"lora_path": lora_name,
|
||||
"target": target,
|
||||
"strength": strength,
|
||||
}
|
||||
|
||||
# Call API
|
||||
|
||||
@@ -5,7 +5,9 @@ only HTTP, so a change on either side of the payload contract fails here.
|
||||
"""
|
||||
|
||||
import importlib.util
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import types
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
@@ -125,25 +127,56 @@ class _Response:
|
||||
return self._payload
|
||||
|
||||
|
||||
_TMP_FILES = []
|
||||
|
||||
|
||||
def _tmp_file(suffix):
|
||||
"""A temp file path that the autouse fixture below deletes afterwards."""
|
||||
with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as fh:
|
||||
_TMP_FILES.append(fh.name)
|
||||
return fh.name
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _remove_tmp_files():
|
||||
yield
|
||||
while _TMP_FILES:
|
||||
path = _TMP_FILES.pop()
|
||||
if os.path.exists(path):
|
||||
os.remove(path)
|
||||
|
||||
|
||||
def _run_node(**node_kwargs):
|
||||
"""Drive the node through the real client and capture the POST payload."""
|
||||
client = SGLDiffusionServerAPI(base_url="http://127.0.0.1:30010")
|
||||
captured = {}
|
||||
# A real, existing path: the client now checks the server's reported
|
||||
# file_path with os.path.exists before trusting it (see
|
||||
# SGLDiffusionServerAPI._localize_video_result), since on a remote
|
||||
# server that path belongs to a different filesystem.
|
||||
out_path = _tmp_file(".mp4")
|
||||
|
||||
def fake_post(url, json=None, headers=None, timeout=None):
|
||||
captured.update(json)
|
||||
def fake_post(url, json=None, headers=None, timeout=None, files=None, data=None):
|
||||
captured.update(json or data or {})
|
||||
return _Response({"id": "job-1"})
|
||||
|
||||
def fake_get(url, headers=None, timeout=None):
|
||||
def fake_get(url, headers=None, timeout=None, **kwargs):
|
||||
return _Response(
|
||||
{
|
||||
"id": "job-1",
|
||||
"status": "completed",
|
||||
"size": RESOLVED_SIZE,
|
||||
"file_path": "/tmp/out.mp4",
|
||||
"file_path": out_path,
|
||||
}
|
||||
)
|
||||
|
||||
def fake_get_image_path(image):
|
||||
# _image_material_uri reads this file's bytes and then deletes it.
|
||||
path = _tmp_file(".png")
|
||||
with open(path, "wb") as fh:
|
||||
fh.write(b"\x89PNG\r\n\x1a\n" + b"\x00" * 16)
|
||||
return path
|
||||
|
||||
node = SGLDiffusionGenerateH3()
|
||||
with (
|
||||
mock.patch(
|
||||
@@ -156,7 +189,7 @@ def _run_node(**node_kwargs):
|
||||
),
|
||||
mock.patch(
|
||||
f"{PKG}.nodes.get_image_path",
|
||||
side_effect=lambda image: "/tmp/frame.png",
|
||||
side_effect=fake_get_image_path,
|
||||
),
|
||||
):
|
||||
result = node.generate(sgld_client=client, **node_kwargs)
|
||||
@@ -191,7 +224,11 @@ def test_fl2va_maps_keyframes_to_frame_indices():
|
||||
|
||||
assert [c["role"] for c in payload["conditions"]] == ["keyframe", "keyframe"]
|
||||
assert [c["frame_index"] for c in payload["conditions"]] == [0, -1]
|
||||
assert all(c["uri"].startswith("file:///") for c in payload["conditions"])
|
||||
# Images are embedded inline (data: URI), not as a local file:// path
|
||||
# the server may not be able to read.
|
||||
assert all(
|
||||
c["uri"].startswith("data:image/png;base64,") for c in payload["conditions"]
|
||||
)
|
||||
|
||||
|
||||
def test_ref2va_preserves_modality_order_for_prompt_tags():
|
||||
@@ -222,7 +259,7 @@ def test_node_reports_the_server_resolved_canvas():
|
||||
_, (video, video_path) = _run_node(positive_prompt="a cat", task="t2va")
|
||||
|
||||
assert video.get_dimensions() == (1344, 768)
|
||||
assert video_path == "/tmp/out.mp4"
|
||||
assert os.path.exists(video_path)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
@@ -242,9 +279,10 @@ def test_extra_fields_win_over_generic_defaults():
|
||||
"""A model's own field must not be shadowed by a same-named generic default."""
|
||||
client = SGLDiffusionServerAPI(base_url="http://127.0.0.1:30010")
|
||||
captured = {}
|
||||
out_path = _tmp_file(".mp4")
|
||||
|
||||
def fake_post(url, json=None, headers=None, timeout=None):
|
||||
captured.update(json)
|
||||
def fake_post(url, json=None, headers=None, timeout=None, files=None, data=None):
|
||||
captured.update(json or data or {})
|
||||
return _Response({"id": "job-1"})
|
||||
|
||||
with (
|
||||
@@ -255,7 +293,12 @@ def test_extra_fields_win_over_generic_defaults():
|
||||
mock.patch(
|
||||
f"{PKG}.core.server_api.requests.get",
|
||||
side_effect=lambda *a, **k: _Response(
|
||||
{"id": "job-1", "status": "completed", "size": RESOLVED_SIZE}
|
||||
{
|
||||
"id": "job-1",
|
||||
"status": "completed",
|
||||
"size": RESOLVED_SIZE,
|
||||
"file_path": out_path,
|
||||
}
|
||||
),
|
||||
),
|
||||
):
|
||||
|
||||
@@ -0,0 +1,758 @@
|
||||
"""Regression tests for ComfyUI_SGLDiffusion server-mode bugs found in audit.
|
||||
|
||||
Each bug gets a test that fails against the pre-fix code and passes after.
|
||||
These stub ComfyUI/ torch-adjacent modules the same way test_h3_request.py
|
||||
does, so they run without a ComfyUI install or a GPU.
|
||||
"""
|
||||
|
||||
import base64
|
||||
import importlib.util
|
||||
import io
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import types
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from PIL import Image
|
||||
|
||||
PLUGIN_DIR = Path(__file__).resolve().parents[1]
|
||||
PKG = "sgld_server_mode_under_test"
|
||||
|
||||
|
||||
def _install_comfy_stubs() -> None:
|
||||
folder_paths = types.ModuleType("folder_paths")
|
||||
_tmp = tempfile.mkdtemp(prefix="sgld_test_temp_")
|
||||
folder_paths.get_temp_directory = lambda: _tmp
|
||||
folder_paths._diffusion_models_dirs = []
|
||||
folder_paths.folder_names_and_paths = {
|
||||
"diffusion_models": ([], {".safetensors", ".sft"}),
|
||||
"loras": ([], {".safetensors"}),
|
||||
}
|
||||
|
||||
def get_filename_list(name):
|
||||
names = set()
|
||||
for folder in folder_paths.folder_names_and_paths.get(name, ([], set()))[0]:
|
||||
if os.path.isdir(folder):
|
||||
names.update(os.listdir(folder))
|
||||
return sorted(names)
|
||||
|
||||
def get_folder_paths(name):
|
||||
return list(folder_paths.folder_names_and_paths.get(name, ([], set()))[0])
|
||||
|
||||
def get_full_path(name, filename):
|
||||
for folder in get_folder_paths(name):
|
||||
candidate = os.path.join(folder, filename)
|
||||
if os.path.isfile(candidate):
|
||||
return candidate
|
||||
return None
|
||||
|
||||
folder_paths.get_filename_list = get_filename_list
|
||||
folder_paths.get_folder_paths = get_folder_paths
|
||||
folder_paths.get_full_path = get_full_path
|
||||
sys.modules["folder_paths"] = folder_paths
|
||||
|
||||
comfy_api = types.ModuleType("comfy_api")
|
||||
comfy_api_input = types.ModuleType("comfy_api.input")
|
||||
|
||||
class VideoInput:
|
||||
pass
|
||||
|
||||
comfy_api_input.VideoInput = VideoInput
|
||||
comfy_api.input = comfy_api_input
|
||||
sys.modules["comfy_api"] = comfy_api
|
||||
sys.modules["comfy_api.input"] = comfy_api_input
|
||||
# No comfy_api.input_impl: exercises the SGLDVideoInput fallback path.
|
||||
sys.modules.pop("comfy_api.input_impl", None)
|
||||
|
||||
comfy = types.ModuleType("comfy")
|
||||
comfy.model_detection = types.ModuleType("comfy.model_detection")
|
||||
comfy.model_management = types.ModuleType("comfy.model_management")
|
||||
|
||||
comfy_utils = types.ModuleType("comfy.utils")
|
||||
for name in (
|
||||
"calculate_parameters",
|
||||
"load_torch_file",
|
||||
"state_dict_prefix_replace",
|
||||
"unet_to_diffusers",
|
||||
):
|
||||
setattr(comfy_utils, name, lambda *a, **k: None)
|
||||
|
||||
class InterruptProcessingException(Exception):
|
||||
pass
|
||||
|
||||
_interrupt = {"flag": False}
|
||||
|
||||
def throw_exception_if_processing_interrupted():
|
||||
if _interrupt["flag"]:
|
||||
raise InterruptProcessingException()
|
||||
|
||||
comfy.model_management.InterruptProcessingException = InterruptProcessingException
|
||||
comfy.model_management.throw_exception_if_processing_interrupted = (
|
||||
throw_exception_if_processing_interrupted
|
||||
)
|
||||
comfy.model_management._test_interrupt = _interrupt
|
||||
|
||||
class ProgressBar:
|
||||
instances = []
|
||||
|
||||
def __init__(self, total):
|
||||
self.total = total
|
||||
self.values = []
|
||||
ProgressBar.instances.append(self)
|
||||
|
||||
def update_absolute(self, value, total=None, preview=None):
|
||||
self.values.append(value)
|
||||
|
||||
comfy_utils.ProgressBar = ProgressBar
|
||||
comfy.utils = comfy_utils
|
||||
|
||||
comfy_model_patcher = types.ModuleType("comfy.model_patcher")
|
||||
|
||||
class ModelPatcher:
|
||||
def __init__(self, model, load_device, offload_device, size=0, *a, **k):
|
||||
self.model = model
|
||||
self.load_device = load_device
|
||||
self.offload_device = offload_device
|
||||
self.size = size
|
||||
self.patches = {}
|
||||
self.object_patches = {}
|
||||
self.model_options = {}
|
||||
self.backup = {}
|
||||
self.object_patches_backup = {}
|
||||
self.wrappers = {}
|
||||
self.callbacks = {}
|
||||
self.patches_uuid = None
|
||||
self.weight_inplace_update = False
|
||||
|
||||
def add_wrapper_with_key(self, *a, **k):
|
||||
pass
|
||||
|
||||
# ComfyUI: loaded_size() reads the model's own weight counter (0 for
|
||||
# SGLD, whose weights live in the worker); detach() is what
|
||||
# LoadedModel.model_unload always ends in.
|
||||
def loaded_size(self):
|
||||
return getattr(self.model, "model_loaded_weight_memory", 0)
|
||||
|
||||
def detach(self, unpatch_all=True):
|
||||
return self.model
|
||||
|
||||
comfy_model_patcher.ModelPatcher = ModelPatcher
|
||||
comfy.model_patcher = comfy_model_patcher
|
||||
|
||||
comfy_patcher_extension = types.ModuleType("comfy.patcher_extension")
|
||||
|
||||
class WrappersMP:
|
||||
SAMPLER_SAMPLE = "SAMPLER_SAMPLE"
|
||||
|
||||
comfy_patcher_extension.WrappersMP = WrappersMP
|
||||
comfy.patcher_extension = comfy_patcher_extension
|
||||
|
||||
sys.modules["comfy"] = comfy
|
||||
sys.modules["comfy.model_detection"] = comfy.model_detection
|
||||
sys.modules["comfy.model_management"] = comfy.model_management
|
||||
sys.modules["comfy.utils"] = comfy_utils
|
||||
sys.modules["comfy.model_patcher"] = comfy_model_patcher
|
||||
sys.modules["comfy.patcher_extension"] = comfy_patcher_extension
|
||||
return folder_paths
|
||||
|
||||
|
||||
def _load(module_name: str, relative_path: str):
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
module_name, PLUGIN_DIR / relative_path
|
||||
)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[module_name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def _load_plugin():
|
||||
folder_paths = _install_comfy_stubs()
|
||||
|
||||
package = types.ModuleType(PKG)
|
||||
package.__path__ = [str(PLUGIN_DIR)]
|
||||
sys.modules[PKG] = package
|
||||
|
||||
core = types.ModuleType(f"{PKG}.core")
|
||||
core.__path__ = [str(PLUGIN_DIR / "core")]
|
||||
sys.modules[f"{PKG}.core"] = core
|
||||
|
||||
server_api = _load(f"{PKG}.core.server_api", "core/server_api.py")
|
||||
core.SGLDiffusionServerAPI = server_api.SGLDiffusionServerAPI
|
||||
model_patcher = _load(f"{PKG}.core.model_patcher", "core/model_patcher.py")
|
||||
core.SGLDModelPatcher = model_patcher.SGLDModelPatcher
|
||||
generator = _load(f"{PKG}.core.generator", "core/generator.py")
|
||||
core.SGLDiffusionGenerator = generator.SGLDiffusionGenerator
|
||||
|
||||
_load(f"{PKG}.utils", "utils.py")
|
||||
nodes = _load(f"{PKG}.nodes", "nodes.py")
|
||||
return (
|
||||
server_api,
|
||||
model_patcher,
|
||||
generator,
|
||||
nodes,
|
||||
sys.modules[f"{PKG}.utils"],
|
||||
folder_paths,
|
||||
)
|
||||
|
||||
|
||||
SERVER_API, MODEL_PATCHER, GENERATOR, NODES, UTILS, FOLDER_PATHS = _load_plugin()
|
||||
SGLDiffusionServerAPI = SERVER_API.SGLDiffusionServerAPI
|
||||
SGLDModelPatcher = MODEL_PATCHER.SGLDModelPatcher
|
||||
|
||||
|
||||
class _Response:
|
||||
def __init__(self, payload, status_code=200):
|
||||
self._payload = payload
|
||||
self.status_code = status_code
|
||||
|
||||
def raise_for_status(self):
|
||||
pass
|
||||
|
||||
def json(self):
|
||||
return self._payload
|
||||
|
||||
def iter_content(self, chunk_size=None):
|
||||
yield b"fake-mp4-bytes"
|
||||
|
||||
|
||||
# --- Bug 9: hardcoded 300s timeout on image requests -----------------------
|
||||
|
||||
|
||||
_TMP_FILES = []
|
||||
|
||||
|
||||
def _tmp_file(suffix):
|
||||
"""A temp file path that the autouse fixture below deletes afterwards."""
|
||||
with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as fh:
|
||||
_TMP_FILES.append(fh.name)
|
||||
return fh.name
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _remove_tmp_files():
|
||||
yield
|
||||
while _TMP_FILES:
|
||||
path = _TMP_FILES.pop()
|
||||
if os.path.exists(path):
|
||||
os.remove(path)
|
||||
|
||||
|
||||
def test_image_timeout_is_configurable():
|
||||
client = SGLDiffusionServerAPI(base_url="http://127.0.0.1:1234", image_timeout=900)
|
||||
captured = {}
|
||||
|
||||
def fake_post(url, json=None, headers=None, timeout=None, **kwargs):
|
||||
captured["timeout"] = timeout
|
||||
return _Response({"data": [{"b64_json": "x"}]})
|
||||
|
||||
with mock.patch(f"{PKG}.core.server_api.requests.post", side_effect=fake_post):
|
||||
client.generate_image(prompt="a cat")
|
||||
|
||||
assert captured["timeout"] == 900
|
||||
|
||||
|
||||
# --- Bug 10: batched image inputs were silently truncated to image[0] ------
|
||||
|
||||
|
||||
def test_get_image_path_uses_first_image_of_a_batch_with_a_warning(caplog):
|
||||
batch = torch.zeros(3, 8, 8, 3)
|
||||
with caplog.at_level("WARNING"):
|
||||
path = UTILS.get_image_path(batch)
|
||||
assert os.path.exists(path)
|
||||
os.remove(path)
|
||||
assert "batch of 3 images" in caplog.text
|
||||
|
||||
|
||||
def test_get_image_path_accepts_single_image():
|
||||
single = torch.zeros(1, 8, 8, 3)
|
||||
path = UTILS.get_image_path(single)
|
||||
assert os.path.exists(path)
|
||||
os.remove(path)
|
||||
|
||||
|
||||
# --- Bug 11: LoRA strength could not be set in server mode -----------------
|
||||
|
||||
|
||||
def test_set_lora_sends_strength_to_server():
|
||||
client = SGLDiffusionServerAPI(base_url="http://127.0.0.1:1234")
|
||||
captured = {}
|
||||
|
||||
def fake_post(url, json=None, headers=None, timeout=None, **kwargs):
|
||||
captured.update(json)
|
||||
return _Response({"status": "ok"})
|
||||
|
||||
with mock.patch(f"{PKG}.core.server_api.requests.post", side_effect=fake_post):
|
||||
client.set_lora(
|
||||
lora_nickname="style", lora_path="style.safetensors", strength=0.7
|
||||
)
|
||||
|
||||
assert captured["strength"] == 0.7
|
||||
|
||||
|
||||
def test_set_lora_node_exposes_strength(monkeypatch):
|
||||
client = SGLDiffusionServerAPI(base_url="http://127.0.0.1:1234")
|
||||
node = NODES.SGLDiffusionServerSetLora()
|
||||
captured = {}
|
||||
|
||||
def fake_set_lora(**kwargs):
|
||||
captured.update(kwargs)
|
||||
return {"status": "ok"}
|
||||
|
||||
monkeypatch.setattr(client, "set_lora", fake_set_lora)
|
||||
node.set_lora(sgld_client=client, lora_name="style.safetensors", strength=0.5)
|
||||
assert captured["strength"] == 0.5
|
||||
|
||||
|
||||
# --- Bug 6/7: SGLDVideoInput didn't implement VideoInput; dims could be wrong
|
||||
|
||||
|
||||
def test_video_input_fallback_fails_loudly_instead_of_returning_wrong_data():
|
||||
video = UTILS.convert_video_to_comfy_video("/tmp/out.mp4", height=720, width=1280)
|
||||
assert isinstance(video, UTILS.SGLDVideoInput) # no comfy_api.input_impl stubbed
|
||||
with pytest.raises(NotImplementedError):
|
||||
video.get_components()
|
||||
|
||||
|
||||
def test_video_input_save_to_refuses_silent_format_drop():
|
||||
video = UTILS.SGLDVideoInput("/tmp/out.mp4", height=720, width=1280)
|
||||
with pytest.raises(NotImplementedError):
|
||||
video.save_to("/tmp/converted.webm", format="webm")
|
||||
|
||||
|
||||
def test_generic_video_node_reports_server_resolved_size_not_requested_size():
|
||||
client = SGLDiffusionServerAPI(base_url="http://127.0.0.1:1234")
|
||||
node = NODES.SGLDiffusionGenerateVideo()
|
||||
|
||||
with (
|
||||
mock.patch.object(
|
||||
client,
|
||||
"generate_video",
|
||||
return_value={"file_path": "/tmp/x.mp4", "size": "1280x704"},
|
||||
),
|
||||
):
|
||||
video, _, _ = node.generate_video(
|
||||
sgld_client=client,
|
||||
positive_prompt="a cat",
|
||||
width=1280,
|
||||
height=720, # requested 720, server actually resolved to 704
|
||||
)
|
||||
assert video.get_dimensions() == (1280, 704)
|
||||
|
||||
|
||||
# --- Bug 3: server-mode nodes sent local paths a remote server can't read --
|
||||
|
||||
|
||||
def test_generate_video_uploads_local_input_reference_instead_of_sending_path():
|
||||
client = SGLDiffusionServerAPI(base_url="http://127.0.0.1:1234")
|
||||
local_image = _tmp_file(".png")
|
||||
with open(local_image, "wb") as fh:
|
||||
fh.write(b"fake-png-bytes")
|
||||
|
||||
seen = {}
|
||||
|
||||
def fake_post(url, json=None, headers=None, timeout=None, files=None, data=None):
|
||||
seen["files"] = files
|
||||
seen["data"] = data
|
||||
seen["json"] = json
|
||||
return _Response({"id": "job-1"})
|
||||
|
||||
out_path = _tmp_file(".mp4")
|
||||
|
||||
def fake_get(url, headers=None, timeout=None, **kwargs):
|
||||
return _Response({"id": "job-1", "status": "completed", "file_path": out_path})
|
||||
|
||||
with (
|
||||
mock.patch(f"{PKG}.core.server_api.requests.post", side_effect=fake_post),
|
||||
mock.patch(f"{PKG}.core.server_api.requests.get", side_effect=fake_get),
|
||||
):
|
||||
client.generate_video(prompt="a cat", input_reference=local_image)
|
||||
|
||||
os.remove(local_image)
|
||||
# A local path must be uploaded as file bytes, not sent as a path string
|
||||
# the server (possibly on another machine) has no way to read.
|
||||
assert seen["files"] is not None
|
||||
assert seen["json"] is None
|
||||
|
||||
|
||||
def _capture_video_request(client, **kwargs):
|
||||
seen = {}
|
||||
|
||||
def fake_post(url, json=None, headers=None, timeout=None, files=None, data=None):
|
||||
seen["files"] = files and {
|
||||
key: (name, fh.read(), content_type)
|
||||
for key, (name, fh, content_type) in files.items()
|
||||
}
|
||||
seen["data"] = data
|
||||
seen["json"] = json
|
||||
return _Response({"id": "job-1"})
|
||||
|
||||
out_path = _tmp_file(".mp4")
|
||||
|
||||
def fake_get(url, headers=None, timeout=None, **kw):
|
||||
return _Response({"id": "job-1", "status": "completed", "file_path": out_path})
|
||||
|
||||
with (
|
||||
mock.patch(f"{PKG}.core.server_api.requests.post", side_effect=fake_post),
|
||||
mock.patch(f"{PKG}.core.server_api.requests.get", side_effect=fake_get),
|
||||
):
|
||||
client.generate_video(prompt="a cat", **kwargs)
|
||||
return seen
|
||||
|
||||
|
||||
def test_generate_video_uploads_local_source_video_as_video_reference():
|
||||
client = SGLDiffusionServerAPI(base_url="http://127.0.0.1:1234")
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
clip = os.path.join(tmp, "clip.mp4")
|
||||
image = os.path.join(tmp, "ref.png")
|
||||
for path, body in ((clip, b"fake-mp4"), (image, b"fake-png")):
|
||||
with open(path, "wb") as fh:
|
||||
fh.write(body)
|
||||
|
||||
seen = _capture_video_request(client, video_path=clip, input_reference=image)
|
||||
|
||||
# Both local files go up as multipart; neither path string is sent.
|
||||
assert seen["json"] is None
|
||||
assert seen["files"]["video_reference"] == ("clip.mp4", b"fake-mp4", "video/mp4")
|
||||
assert seen["files"]["input_reference"][1] == b"fake-png"
|
||||
assert "video_path" not in seen["data"]
|
||||
assert "input_reference" not in seen["data"]
|
||||
|
||||
|
||||
def test_generate_video_multipart_fields_are_json_encoded():
|
||||
client = SGLDiffusionServerAPI(base_url="http://127.0.0.1:1234")
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
image = os.path.join(tmp, "ref.png")
|
||||
with open(image, "wb") as fh:
|
||||
fh.write(b"fake-png")
|
||||
seen = _capture_video_request(
|
||||
client,
|
||||
input_reference=image,
|
||||
extra_fields={"target": {"type": "image"}, "flag": False},
|
||||
)
|
||||
|
||||
# the server json-decodes form values, so Python repr would break them
|
||||
assert seen["data"]["target"] == '{"type": "image"}'
|
||||
assert seen["data"]["flag"] == "false"
|
||||
assert seen["data"]["prompt"] == "a cat"
|
||||
|
||||
|
||||
def test_generate_video_sends_remote_source_video_as_video_path():
|
||||
client = SGLDiffusionServerAPI(base_url="http://127.0.0.1:1234")
|
||||
for source in ("https://example.com/clip.mp4", "/srv/sglang/inputs/clip.mp4"):
|
||||
seen = _capture_video_request(client, video_path=source)
|
||||
# A URL or a path that only exists on the server stays a string.
|
||||
assert seen["files"] is None
|
||||
assert seen["json"]["video_path"] == source
|
||||
|
||||
|
||||
def test_generate_video_downloads_result_when_server_path_is_not_local():
|
||||
client = SGLDiffusionServerAPI(base_url="http://127.0.0.1:1234")
|
||||
|
||||
def fake_post(url, **kwargs):
|
||||
return _Response({"id": "job-1"})
|
||||
|
||||
def fake_get(url, headers=None, timeout=None, stream=False, **kwargs):
|
||||
if url.endswith("/content"):
|
||||
return _Response({}, status_code=200)
|
||||
return _Response(
|
||||
{
|
||||
"id": "job-1",
|
||||
"status": "completed",
|
||||
# A path on the *server's* filesystem, not this one.
|
||||
"file_path": "/srv/sglang/outputs/job-1.mp4",
|
||||
}
|
||||
)
|
||||
|
||||
with tempfile.TemporaryDirectory() as out_dir:
|
||||
with (
|
||||
mock.patch(f"{PKG}.core.server_api.requests.post", side_effect=fake_post),
|
||||
mock.patch(f"{PKG}.core.server_api.requests.get", side_effect=fake_get),
|
||||
):
|
||||
result = client.generate_video(prompt="a cat", output_path=out_dir)
|
||||
|
||||
assert os.path.exists(result["file_path"])
|
||||
assert os.path.realpath(
|
||||
os.path.dirname(result["file_path"])
|
||||
) == os.path.realpath(out_dir)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("video_id", ["../escape", "a/b", "job.1", ""])
|
||||
def test_localize_video_result_rejects_unsafe_video_id(video_id):
|
||||
client = SGLDiffusionServerAPI(base_url="http://127.0.0.1:1234")
|
||||
status = {"status": "completed", "file_path": "/srv/out/x.mp4"}
|
||||
with tempfile.TemporaryDirectory() as out_dir:
|
||||
with mock.patch(f"{PKG}.core.server_api.requests.get") as get:
|
||||
with pytest.raises(RuntimeError, match="unsafe id"):
|
||||
client._localize_video_result(status, video_id, out_dir)
|
||||
get.assert_not_called()
|
||||
assert os.listdir(out_dir) == []
|
||||
|
||||
|
||||
def test_localize_video_result_whitelists_extension_and_cleans_up_on_error():
|
||||
client = SGLDiffusionServerAPI(base_url="http://127.0.0.1:1234")
|
||||
|
||||
class _Broken(_Response):
|
||||
def iter_content(self, chunk_size=None):
|
||||
yield b"partial"
|
||||
raise SERVER_API.requests.exceptions.ConnectionError("dropped")
|
||||
|
||||
status = {"status": "completed", "file_path": "/srv/out/job-1.sh"}
|
||||
with tempfile.TemporaryDirectory() as out_dir:
|
||||
with mock.patch(
|
||||
f"{PKG}.core.server_api.requests.get", return_value=_Response({})
|
||||
):
|
||||
result = client._localize_video_result(status, "job-1", out_dir)
|
||||
assert result["file_path"] == os.path.join(
|
||||
os.path.realpath(out_dir), "job-1.mp4"
|
||||
)
|
||||
os.remove(result["file_path"])
|
||||
with mock.patch(
|
||||
f"{PKG}.core.server_api.requests.get", return_value=_Broken({})
|
||||
):
|
||||
with pytest.raises(SERVER_API.requests.exceptions.ConnectionError):
|
||||
client._localize_video_result(status, "job-1", out_dir)
|
||||
assert os.listdir(out_dir) == []
|
||||
|
||||
|
||||
class _HTTPErrorResponse(_Response):
|
||||
def raise_for_status(self):
|
||||
raise SERVER_API.requests.exceptions.HTTPError(
|
||||
f"{self.status_code} Client Error", response=self
|
||||
)
|
||||
|
||||
|
||||
def _completed_video_client(job, max_polls=3):
|
||||
"""Client whose job is already completed; /content answers 400, as the
|
||||
server does for a job uploaded to cloud storage. Polling past
|
||||
`max_polls` fails the test instead of looping until max_wait_time."""
|
||||
client = SGLDiffusionServerAPI(base_url="http://127.0.0.1:1234")
|
||||
calls = {"status": 0, "content": 0}
|
||||
|
||||
def fake_get(url, **kwargs):
|
||||
if url.endswith("/content"):
|
||||
calls["content"] += 1
|
||||
return _HTTPErrorResponse({"detail": "use the cloud URL"}, 400)
|
||||
calls["status"] += 1
|
||||
assert calls["status"] <= max_polls, "kept polling a completed job"
|
||||
return _Response(job)
|
||||
|
||||
def fake_post(url, **kwargs):
|
||||
return _Response({"id": "job-1"})
|
||||
|
||||
patches = (
|
||||
mock.patch.object(SERVER_API.requests, "get", fake_get),
|
||||
mock.patch.object(SERVER_API.requests, "post", fake_post),
|
||||
mock.patch.object(SERVER_API.time, "sleep"),
|
||||
)
|
||||
return client, calls, patches
|
||||
|
||||
|
||||
def test_generate_video_returns_cloud_url_without_downloading_content():
|
||||
url = "https://bucket.example/job-1.mp4"
|
||||
client, calls, patches = _completed_video_client(
|
||||
{"id": "job-1", "status": "completed", "url": url, "file_path": None}
|
||||
)
|
||||
with tempfile.TemporaryDirectory() as out_dir:
|
||||
with patches[0], patches[1], patches[2]:
|
||||
result = client.generate_video(prompt="a cat", output_path=out_dir)
|
||||
# resolve_video_path fetches the url, as on main.
|
||||
assert result["url"] == url
|
||||
assert result["file_path"] is None
|
||||
assert calls["content"] == 0
|
||||
|
||||
|
||||
def test_generate_video_fails_fast_when_result_download_fails():
|
||||
client, calls, patches = _completed_video_client(
|
||||
{"id": "job-1", "status": "completed", "file_path": "/srv/out/job-1.mp4"}
|
||||
)
|
||||
with tempfile.TemporaryDirectory() as out_dir:
|
||||
with patches[0], patches[1], patches[2]:
|
||||
with pytest.raises(RuntimeError, match="400"):
|
||||
client.generate_video(prompt="a cat", output_path=out_dir)
|
||||
assert calls == {"status": 1, "content": 1}
|
||||
|
||||
|
||||
def test_h3_image_conditions_use_data_uri_not_local_file_path():
|
||||
node = NODES.SGLDiffusionGenerateH3()
|
||||
image = torch.zeros(1, 8, 8, 3)
|
||||
uri = node._image_material_uri(image)
|
||||
assert uri.startswith("data:image/png;base64,")
|
||||
|
||||
|
||||
def test_h3_rejects_local_video_reference_against_remote_server():
|
||||
with pytest.raises(ValueError, match="no way to read or upload"):
|
||||
NODES.SGLDiffusionGenerateH3._remote_material_uri(
|
||||
"/data/clip.mp4", "reference_video", "http://remote-host.example:3000/v1"
|
||||
)
|
||||
|
||||
|
||||
def test_h3_allows_local_video_reference_against_local_server():
|
||||
uri = NODES.SGLDiffusionGenerateH3._remote_material_uri(
|
||||
"/data/clip.mp4", "reference_video", "http://127.0.0.1:3000/v1"
|
||||
)
|
||||
assert uri == "file:///data/clip.mp4"
|
||||
|
||||
|
||||
def test_is_local_server_accepts_own_hostname_and_loopback():
|
||||
is_local = NODES.SGLDiffusionGenerateH3._is_local_server
|
||||
host = NODES.socket.gethostname()
|
||||
assert is_local(f"http://{host}:3000/v1")
|
||||
assert is_local("http://0.0.0.0:3000/v1")
|
||||
assert is_local("http://127.0.0.2:3000/v1")
|
||||
assert is_local("http://[::1]:3000/v1")
|
||||
with mock.patch.object(
|
||||
NODES.socket, "getaddrinfo", return_value=[(0, 0, 0, "", ("10.1.2.3", 0))]
|
||||
):
|
||||
assert not is_local("http://gpu-box:3000/v1")
|
||||
|
||||
|
||||
def test_h3_passes_file_uri_through_for_any_server():
|
||||
# a file:// URI names a path on the server, so it is never rewritten
|
||||
for base_url in ("http://127.0.0.1:3000/v1", "http://remote-host.example:3000/v1"):
|
||||
uri = NODES.SGLDiffusionGenerateH3._remote_material_uri(
|
||||
"file:///srv/clip.mp4", "reference_video", base_url
|
||||
)
|
||||
assert uri == "file:///srv/clip.mp4"
|
||||
|
||||
|
||||
# --- server-mode video jobs: no progress bar, Cancel was ignored ------------
|
||||
|
||||
|
||||
def _video_client_with_statuses(statuses):
|
||||
client = SGLDiffusionServerAPI(base_url="http://127.0.0.1:1234")
|
||||
polled = iter(statuses)
|
||||
|
||||
def fake_get(url, **kwargs):
|
||||
return _Response(next(polled))
|
||||
|
||||
def fake_post(url, **kwargs):
|
||||
return _Response({"id": "vid1"})
|
||||
|
||||
return client, fake_get, fake_post
|
||||
|
||||
|
||||
def test_generate_video_reports_server_progress():
|
||||
client, fake_get, fake_post = _video_client_with_statuses(
|
||||
[
|
||||
{"status": "queued", "progress": 0},
|
||||
{"status": "in_progress", "progress": 40},
|
||||
{"status": "completed", "progress": 100, "file_path": "/tmp/x.mp4"},
|
||||
]
|
||||
)
|
||||
seen = []
|
||||
with (
|
||||
mock.patch.object(SERVER_API.requests, "get", fake_get),
|
||||
mock.patch.object(SERVER_API.requests, "post", fake_post),
|
||||
mock.patch.object(SERVER_API.time, "sleep"),
|
||||
mock.patch.object(client, "_localize_video_result", lambda s, *a: s),
|
||||
):
|
||||
client.generate_video(prompt="a cat", progress_callback=seen.append)
|
||||
assert seen == [0, 40, 100]
|
||||
|
||||
|
||||
def test_generate_video_stops_polling_when_interrupted():
|
||||
client, fake_get, fake_post = _video_client_with_statuses(
|
||||
[{"status": "in_progress", "progress": 10}] * 50
|
||||
)
|
||||
checks = {"n": 0}
|
||||
|
||||
class Interrupted(Exception):
|
||||
pass
|
||||
|
||||
def check_interrupt():
|
||||
checks["n"] += 1
|
||||
if checks["n"] > 2:
|
||||
raise Interrupted()
|
||||
|
||||
with (
|
||||
mock.patch.object(SERVER_API.requests, "get", fake_get),
|
||||
mock.patch.object(SERVER_API.requests, "post", fake_post),
|
||||
mock.patch.object(SERVER_API.time, "sleep"),
|
||||
):
|
||||
with pytest.raises(Interrupted):
|
||||
client.generate_video(prompt="a cat", check_interrupt=check_interrupt)
|
||||
assert checks["n"] == 3 # stopped promptly, did not poll to completion
|
||||
|
||||
|
||||
def test_video_node_wires_comfy_progress_bar_and_interrupt():
|
||||
client = SGLDiffusionServerAPI(base_url="http://127.0.0.1:1234")
|
||||
node = NODES.SGLDiffusionGenerateVideo()
|
||||
captured = {}
|
||||
|
||||
def fake_generate_video(**kwargs):
|
||||
captured.update(kwargs)
|
||||
return {"file_path": "/tmp/x.mp4", "size": "1280x720"}
|
||||
|
||||
with mock.patch.object(client, "generate_video", fake_generate_video):
|
||||
node.generate_video(
|
||||
sgld_client=client, positive_prompt="a cat", width=1280, height=720
|
||||
)
|
||||
captured["progress_callback"](55)
|
||||
assert sys.modules["comfy.utils"].ProgressBar.instances[-1].values == [55]
|
||||
sys.modules["comfy.model_management"]._test_interrupt["flag"] = True
|
||||
try:
|
||||
with pytest.raises(
|
||||
sys.modules["comfy.model_management"].InterruptProcessingException
|
||||
):
|
||||
captured["check_interrupt"]()
|
||||
finally:
|
||||
sys.modules["comfy.model_management"]._test_interrupt["flag"] = False
|
||||
|
||||
|
||||
@pytest.mark.parametrize("enhance", [True, False])
|
||||
def test_image_node_enhance_prompt_returns_revised_prompt(enhance):
|
||||
client = SGLDiffusionServerAPI(base_url="http://127.0.0.1:1234")
|
||||
png = io.BytesIO()
|
||||
Image.new("RGB", (8, 8)).save(png, format="PNG")
|
||||
b64 = base64.b64encode(png.getvalue()).decode()
|
||||
revised = "a fluffy cat" if enhance else None
|
||||
captured = {}
|
||||
|
||||
def fake_post(url, json=None, **kwargs):
|
||||
captured.update(json)
|
||||
return _Response({"data": [{"b64_json": b64, "revised_prompt": revised}]})
|
||||
|
||||
with mock.patch(f"{PKG}.core.server_api.requests.post", side_effect=fake_post):
|
||||
image, revised_prompt = NODES.SGLDiffusionGenerateImage().generate_image(
|
||||
sgld_client=client, positive_prompt="a cat", enhance_prompt=enhance
|
||||
)
|
||||
|
||||
assert captured.get("enhance_prompt", False) == enhance
|
||||
assert image.shape == (1, 8, 8, 3)
|
||||
assert revised_prompt == ("a fluffy cat" if enhance else "a cat")
|
||||
|
||||
|
||||
def test_video_node_sends_source_video_and_enhance_prompt():
|
||||
client = SGLDiffusionServerAPI(base_url="http://127.0.0.1:1234")
|
||||
captured = {}
|
||||
|
||||
class SourceVideo:
|
||||
def save_to(self, path):
|
||||
with open(path, "wb") as fh:
|
||||
fh.write(b"source-mp4")
|
||||
|
||||
def fake_generate_video(**kwargs):
|
||||
captured.update(kwargs)
|
||||
with open(kwargs["video_path"], "rb") as fh:
|
||||
captured["video_bytes"] = fh.read()
|
||||
return {"file_path": "/tmp/x.mp4", "revised_prompt": "a fluffy cat"}
|
||||
|
||||
with mock.patch.object(client, "generate_video", fake_generate_video):
|
||||
_, video_path, revised_prompt = (
|
||||
NODES.SGLDiffusionGenerateVideo().generate_video(
|
||||
sgld_client=client,
|
||||
positive_prompt="a cat",
|
||||
video=SourceVideo(),
|
||||
enhance_prompt=True,
|
||||
)
|
||||
)
|
||||
os.remove(captured["video_path"])
|
||||
|
||||
assert captured["video_bytes"] == b"source-mp4"
|
||||
assert captured["extra_fields"] == {"enhance_prompt": True}
|
||||
assert (video_path, revised_prompt) == ("/tmp/x.mp4", "a fluffy cat")
|
||||
@@ -1,5 +1,6 @@
|
||||
import base64
|
||||
import io
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
@@ -13,6 +14,8 @@ import torch
|
||||
from comfy_api.input import VideoInput
|
||||
from PIL import Image
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _ensure_dir(path: str) -> None:
|
||||
os.makedirs(path, exist_ok=True)
|
||||
@@ -82,7 +85,16 @@ def is_empty_image(image: torch.Tensor, tolerance: float = 1e-6) -> bool:
|
||||
def get_image_path(image: torch.Tensor) -> str:
|
||||
"""
|
||||
Save tensor image to ComfyUI temp directory as PNG and return the path.
|
||||
|
||||
This input only has a single image slot, so a batch keeps its first
|
||||
image (as before) and logs a warning that the rest are dropped.
|
||||
"""
|
||||
if image.dim() == 4 and image.shape[0] > 1:
|
||||
logger.warning(
|
||||
"get_image_path received a batch of %d images but this input only "
|
||||
"accepts one; using the first. Use 'Image From Batch' to pick another.",
|
||||
image.shape[0],
|
||||
)
|
||||
temp_dir = folder_paths.get_temp_directory()
|
||||
|
||||
# Build file name
|
||||
@@ -131,6 +143,14 @@ def convert_b64_to_tensor_image(b64_image: str) -> torch.Tensor:
|
||||
|
||||
|
||||
class SGLDVideoInput(VideoInput):
|
||||
"""Minimal fallback used only when ComfyUI's own file-backed VideoInput
|
||||
isn't importable. It cannot decode frames/audio or re-encode on save, so
|
||||
it fails loudly on those rather than silently returning wrong data (the
|
||||
previous ``get_components`` returned ``[video_path]`` instead of a
|
||||
``VideoComponents``, and ``save_to`` dropped ``format``/``codec``/
|
||||
``metadata`` without telling the caller).
|
||||
"""
|
||||
|
||||
def __init__(self, video_path: str, height: int, width: int):
|
||||
super().__init__()
|
||||
|
||||
@@ -148,16 +168,19 @@ class SGLDVideoInput(VideoInput):
|
||||
return self.width, self.height
|
||||
|
||||
def get_components(self):
|
||||
"""
|
||||
Returns the components of the video input.
|
||||
This is required by the VideoInput abstract base class.
|
||||
"""
|
||||
return [self.video_path]
|
||||
raise NotImplementedError(
|
||||
"SGLDVideoInput cannot decode frames/audio (no ComfyUI "
|
||||
"VideoFromFile available in this environment); pass the file "
|
||||
"at video_path to a node that reads files directly instead."
|
||||
)
|
||||
|
||||
def save_to(self, path: str, format=None, codec=None, metadata=None):
|
||||
"""
|
||||
Abstract method to save the video input to a file.
|
||||
"""
|
||||
if format is not None or codec is not None or metadata is not None:
|
||||
raise NotImplementedError(
|
||||
"SGLDVideoInput can only copy the file as-is; format/codec/"
|
||||
"metadata conversion needs ComfyUI's own VideoFromFile, "
|
||||
"which is unavailable in this environment."
|
||||
)
|
||||
if not os.path.exists(self.video_path):
|
||||
raise FileNotFoundError(
|
||||
f"Video not found at '{self.video_path}'. This file must be "
|
||||
@@ -236,7 +259,18 @@ def convert_video_to_comfy_video(
|
||||
video_path: str, height: int, width: int
|
||||
) -> VideoInput:
|
||||
"""
|
||||
Convert video to ComfyUI VIDEO format (VideoInput).
|
||||
Convert a video file on disk to ComfyUI's VIDEO type.
|
||||
|
||||
Prefers ComfyUI's own file-backed ``VideoFromFile``: it probes the real
|
||||
file for dimensions/frames/audio, so it is also immune to the
|
||||
width/height we were told (which can be wrong, e.g. when the server
|
||||
picked a different size than requested to match an image-to-video
|
||||
input). ``height``/``width`` are only used by the ``SGLDVideoInput``
|
||||
fallback below.
|
||||
"""
|
||||
video_input = SGLDVideoInput(video_path, height, width)
|
||||
return video_input
|
||||
try:
|
||||
from comfy_api.input_impl import VideoFromFile
|
||||
|
||||
return VideoFromFile(video_path)
|
||||
except ImportError:
|
||||
return SGLDVideoInput(video_path, height, width)
|
||||
|
||||
@@ -33,7 +33,13 @@ def _load_nodes():
|
||||
comfy_api_input = types.ModuleType("comfy_api.input")
|
||||
comfy_api_input.VideoInput = type("VideoInput", (), {})
|
||||
comfy_api.input = comfy_api_input
|
||||
comfy = types.ModuleType("comfy")
|
||||
comfy.model_management = types.ModuleType("comfy.model_management")
|
||||
comfy.utils = types.ModuleType("comfy.utils")
|
||||
stubs = {
|
||||
"comfy": comfy,
|
||||
"comfy.model_management": comfy.model_management,
|
||||
"comfy.utils": comfy.utils,
|
||||
"folder_paths": folder_paths,
|
||||
"comfy_api": comfy_api,
|
||||
"comfy_api.input": comfy_api_input,
|
||||
|
||||
Reference in New Issue
Block a user