diff --git a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/README.md b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/README.md index 57d91940ba..4b14758ad6 100644 --- a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/README.md +++ b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/README.md @@ -34,7 +34,8 @@ Connect to a standalone SGLang Diffusion server. 2. **Connect to Server**: Use the `SGLDiffusion Server Model` node to connect (default: `http://localhost:3000/v1`). 3. **Generate Content**: - `SGLDiffusion Generate Image`: For text-to-image and image editing. Set `num_images` (1 to 10) to get several images from one request as one `IMAGE` batch, in order. Image `i` uses `seed + i`, so the same settings and seed reproduce the same batch; a seed of `-1` lets the server pick a random seed. - - `SGLDiffusion Generate Video`: For text-to-video and image-to-video. + - `SGLDiffusion Generate Video`: For text-to-video, image-to-video, and video-to-video (connect a `VIDEO`). + - Both nodes have an `enhance_prompt` toggle (the server needs `--prompt-enhancer-config`); the rewritten prompt comes out as `revised_prompt`. - `SGLDiffusion Generate MiniMax-H3`: For joint video-and-audio on a standalone server. 4. **LoRA Support**: Use `SGLDiffusion Server Set LoRA` and `SGLDiffusion Server Unset LoRA`. diff --git a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/core/server_api.py b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/core/server_api.py index 33e287dd39..04721e6622 100644 --- a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/core/server_api.py +++ b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/core/server_api.py @@ -3,24 +3,43 @@ SGLang Diffusion Server API client. Provides a low-level interface for interacting with SGLang Diffusion HTTP server. """ +import contextlib +import json import os +import re import time -from typing import Any, Dict, Optional +from typing import Any, Callable, Dict, Optional import requests +# Server-supplied names used to build a local path for a downloaded video. +_SAFE_VIDEO_ID = re.compile(r"[A-Za-z0-9_-]+") +_VIDEO_EXTENSIONS = (".mp4", ".webm", ".mov", ".mkv") +_MAX_VIDEO_BYTES = 2 * 1024**3 +_DOWNLOAD_DEADLINE_S = 600 + class SGLDiffusionServerAPI: """Client for SGLang Diffusion HTTP server API.""" - def __init__(self, base_url: str, api_key: str = "sk-proj-1234567890"): + def __init__( + self, + base_url: str, + api_key: str = "sk-proj-1234567890", + image_timeout: int = 300, + ): """ Initialize the API client. Args: base_url: Base URL of the SGLang Diffusion server (e.g., "http://localhost:30010/v1") api_key: API key for authentication (default: "sk-proj-1234567890") + image_timeout: Seconds to wait for an image generate/edit request + before timing out. A large job (big size, or a first request + that triggers torch.compile/warmup) can need more than the + previous hardcoded 300s. """ + self.image_timeout = image_timeout # Ensure base_url doesn't end with /v1 if it's already there if base_url.endswith("/v1"): self.base_url = base_url @@ -80,6 +99,7 @@ class SGLDiffusionServerAPI: background: Optional[str] = "auto", output_format: Optional[str] = None, generator_device: Optional[str] = "cuda", + enhance_prompt: bool = False, ) -> Dict[str, Any]: """ Generate or edit an image using SGLang Diffusion API. @@ -105,6 +125,7 @@ class SGLDiffusionServerAPI: background: Background type ("auto", "transparent", "opaque") output_format: Output format ("png", "jpeg", "webp") generator_device: Device for random generator ("cuda" or "cpu") + enhance_prompt: Rewrite the prompt on the server first Returns: Dictionary containing the API response with generated/edited image data @@ -133,6 +154,7 @@ class SGLDiffusionServerAPI: background=background, output_format=output_format, generator_device=generator_device, + enhance_prompt=enhance_prompt, ) # If image_path is provided, use edit endpoint @@ -172,7 +194,7 @@ class SGLDiffusionServerAPI: files=files, data=data, headers=headers, - timeout=300, # 5 minutes timeout for generation + timeout=self.image_timeout, ) response.raise_for_status() return response.json() @@ -196,7 +218,7 @@ class SGLDiffusionServerAPI: f"{self.base_url}/images/generations", json=payload, headers=self.headers, - timeout=300, # 5 minutes timeout for generation + timeout=self.image_timeout, ) response.raise_for_status() return response.json() @@ -219,12 +241,19 @@ class SGLDiffusionServerAPI: enable_teacache: bool = False, generator_device: Optional[str] = "cuda", input_reference: Optional[str] = None, + video_path: Optional[str] = None, output_path: Optional[str] = None, extra_fields: Optional[Dict[str, Any]] = None, + progress_callback: Optional[Callable[[int], None]] = None, + check_interrupt: Optional[Callable[[], None]] = None, ) -> Dict[str, Any]: """ Generate a video using SGLang Diffusion API and wait for completion. + ``progress_callback`` gets the job's ``progress`` (0-100) on each poll. + ``check_interrupt`` is called between polls and may raise to stop + waiting; the server has no abort yet, so the job itself keeps running. + Args: prompt: Text prompt for video generation size: Video size in format "WIDTHxHEIGHT" (e.g., "1280x720") @@ -240,6 +269,8 @@ class SGLDiffusionServerAPI: enable_teacache: Enable TEA cache acceleration generator_device: Device for random generator ("cuda" or "cpu") input_reference: Path to input reference image for image-to-video + video_path: Source clip for video-to-video. A local file is + uploaded; a URL or server-side path is sent as is extra_fields: Model-specific request fields merged into the payload last, so a caller can reach a model's own request surface (MiniMax-H3's `task`/`conditions`/`target`, per-model flow @@ -258,48 +289,89 @@ class SGLDiffusionServerAPI: else: size = "720x1280" - # Prepare request payload - payload: Dict[str, Any] = { + # Prepare request fields shared by the JSON and multipart paths. + # `output_path` is deliberately never sent: it used to be the + # ComfyUI node's own temp directory, which only exists on this + # machine and breaks as soon as the server runs elsewhere. Letting + # the server pick its own output location and fetching the result + # (see below) works for both the local and remote case. + fields: Dict[str, Any] = { "prompt": prompt, "size": size, } - - # Add optional parameters if seconds is not None: - payload["seconds"] = seconds + fields["seconds"] = seconds if fps is not None: - payload["fps"] = fps + fields["fps"] = fps if num_frames is not None: - payload["num_frames"] = num_frames + fields["num_frames"] = num_frames if negative_prompt: - payload["negative_prompt"] = negative_prompt + fields["negative_prompt"] = negative_prompt if guidance_scale is not None: - payload["guidance_scale"] = guidance_scale + fields["guidance_scale"] = guidance_scale if num_inference_steps is not None: - payload["num_inference_steps"] = num_inference_steps + fields["num_inference_steps"] = num_inference_steps if seed is not None and seed >= 0: - payload["seed"] = seed + fields["seed"] = seed if enable_teacache: - payload["enable_teacache"] = True + fields["enable_teacache"] = True if generator_device: - payload["generator_device"] = generator_device - if input_reference: - payload["input_reference"] = input_reference - if output_path: - payload["output_path"] = output_path + fields["generator_device"] = generator_device # merged last so a model-specific field wins over a generic default of # the same name (H3 sizes its output from `target`, not `size`) if extra_fields: - payload.update(extra_fields) + fields.update(extra_fields) + + # Upload local files instead of sending their paths: a server on + # another machine cannot read this machine's filesystem (the images + # endpoint already does this for image_path). The server takes an + # uploaded source clip as `video_reference`. + string_fields: Dict[str, Any] = {} + uploads: Dict[str, str] = {} + for form_key, json_key, path in ( + ("input_reference", "input_reference", input_reference), + ("video_reference", "video_path", video_path), + ): + if not path: + continue + if self._is_local_file(path): + uploads[form_key] = path + else: + string_fields[json_key] = path try: - # Create video generation job - response = requests.post( - f"{self.base_url}/videos", - json=payload, - headers=self.headers, - timeout=30, - ) + if uploads: + # JSON, not repr: the server json-decodes form values, so a + # nested dict or False must not arrive as Python's repr + data = { + key: ( + value if isinstance(value, (str, bytes)) else json.dumps(value) + ) + for key, value in {**fields, **string_fields}.items() + } + with contextlib.ExitStack() as stack: + files = { + form_key: ( + os.path.basename(path), + stack.enter_context(open(path, "rb")), + self._get_content_type(path), + ) + for form_key, path in uploads.items() + } + response = requests.post( + f"{self.base_url}/videos", + files=files, + data=data, + headers={"Authorization": f"Bearer {self.api_key}"}, + timeout=30, + ) + else: + response = requests.post( + f"{self.base_url}/videos", + json={**fields, **string_fields}, + headers=self.headers, + timeout=30, + ) response.raise_for_status() video_job = response.json() video_id = video_job.get("id") @@ -312,6 +384,8 @@ class SGLDiffusionServerAPI: start_time = time.time() while time.time() - start_time < max_wait_time: + if check_interrupt: + check_interrupt() try: status_response = requests.get( f"{self.base_url}/videos/{video_id}", @@ -324,8 +398,13 @@ class SGLDiffusionServerAPI: # Reset error counter on successful request consecutive_errors = 0 + if progress_callback and status.get("progress") is not None: + progress_callback(int(status["progress"])) + if status.get("status") == "completed": - return status + # Localize outside this try, so a failed download + # fails fast instead of being retried as a poll error. + break elif status.get("status") == "failed": error = status.get("error", {}) error_msg = ( @@ -350,14 +429,80 @@ class SGLDiffusionServerAPI: f"Network error after {consecutive_errors} consecutive failures: {str(e)}" ) - time.sleep(poll_interval) - - raise TimeoutError( - f"Video generation timed out after {max_wait_time} seconds" - ) + # Sleep in short slices so Cancel is honoured within ~1s. + for _ in range(poll_interval): + if check_interrupt: + check_interrupt() + time.sleep(1) + else: + raise TimeoutError( + f"Video generation timed out after {max_wait_time} seconds" + ) + return self._localize_video_result(status, video_id, output_path) except requests.exceptions.RequestException as e: raise RuntimeError(f"Failed to generate video: {str(e)}") + def _localize_video_result( + self, status: Dict[str, Any], video_id: str, output_path: Optional[str] + ) -> Dict[str, Any]: + """Make sure ``file_path`` in a completed job points at a local file. + + The server's own ``file_path`` is only readable when the server runs + on this machine. When it isn't (no such local file), download the + result through ``GET /videos/{id}/content`` instead of trusting a + path that belongs to a different filesystem. A job uploaded to cloud + storage has no file_path and a ``url`` instead; ``/content`` refuses + those, so it is returned as is and the node fetches the url. + """ + file_path = status.get("file_path") + if file_path and os.path.exists(file_path): + return status + if not file_path and status.get("url"): + return status + if not isinstance(video_id, str) or not _SAFE_VIDEO_ID.fullmatch(video_id): + raise RuntimeError(f"Refusing to save video with unsafe id {video_id!r}") + ext = os.path.splitext(file_path or "")[1].lower() or ".mp4" + if ext not in _VIDEO_EXTENSIONS: + ext = ".mp4" + target_dir = os.path.realpath(output_path or os.getcwd()) + os.makedirs(target_dir, exist_ok=True) + local_path = os.path.realpath(os.path.join(target_dir, f"{video_id}{ext}")) + if os.path.commonpath([target_dir, local_path]) != target_dir: + raise RuntimeError(f"Refusing to save video outside {target_dir}") + deadline = time.monotonic() + _DOWNLOAD_DEADLINE_S + response = requests.get( + f"{self.base_url}/videos/{video_id}/content", + headers=self.headers, + timeout=300, + stream=True, + ) + try: + response.raise_for_status() + written = 0 + with open(local_path, "wb") as fh: + for chunk in response.iter_content(chunk_size=1024 * 1024): + written += len(chunk) + if written > _MAX_VIDEO_BYTES: + raise RuntimeError( + f"Video download exceeded {_MAX_VIDEO_BYTES} bytes" + ) + if time.monotonic() > deadline: + raise RuntimeError( + f"Video download exceeded {_DOWNLOAD_DEADLINE_S}s" + ) + fh.write(chunk) + except BaseException: + with contextlib.suppress(OSError): + os.remove(local_path) + raise + finally: + close = getattr(response, "close", None) + if close is not None: + close() + status = dict(status) + status["file_path"] = local_path + return status + def _build_image_common_params( self, prompt: str, @@ -372,6 +517,7 @@ class SGLDiffusionServerAPI: background: Optional[str] = None, output_format: Optional[str] = None, generator_device: Optional[str] = None, + enhance_prompt: bool = False, ) -> Dict[str, Any]: """ Build common parameters for both image generation and editing. @@ -403,9 +549,17 @@ class SGLDiffusionServerAPI: params["output_format"] = output_format if generator_device: params["generator_device"] = generator_device + if enhance_prompt: + params["enhance_prompt"] = True return params + @staticmethod + def _is_local_file(path: str) -> bool: + return not path.startswith(("http://", "https://", "data:")) and os.path.isfile( + path + ) + def _get_content_type(self, file_path: str) -> str: """Get content type based on file extension.""" ext = os.path.splitext(file_path)[1].lower() @@ -414,6 +568,10 @@ class SGLDiffusionServerAPI: ".jpg": "image/jpeg", ".jpeg": "image/jpeg", ".webp": "image/webp", + ".mp4": "video/mp4", + ".webm": "video/webm", + ".mov": "video/quicktime", + ".mkv": "video/x-matroska", } return content_types.get(ext, "image/png") @@ -422,6 +580,7 @@ class SGLDiffusionServerAPI: lora_nickname: str, lora_path: Optional[str] = None, target: str = "all", + strength: Optional[float] = None, ) -> Dict[str, Any]: """ Set a LoRA adapter for the specified transformer(s). @@ -435,6 +594,8 @@ class SGLDiffusionServerAPI: - "transformer": Apply only to the primary transformer (high noise for Wan2.2) - "transformer_2": Apply only to transformer_2 (low noise for Wan2.2) - "critic": Apply only to the critic model + strength: LoRA merge strength. The server defaults to 1.0 when + not given; values < 1.0 reduce the effect. Returns: Dictionary containing the API response with status and message @@ -451,6 +612,8 @@ class SGLDiffusionServerAPI: # Add optional lora_path if provided if lora_path: payload["lora_path"] = lora_path + if strength is not None: + payload["strength"] = strength try: response = requests.post( diff --git a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/nodes.py b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/nodes.py index e675a00ba6..01e9449d13 100644 --- a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/nodes.py +++ b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/nodes.py @@ -3,9 +3,15 @@ ComfyUI nodes for SGLang Diffusion integration. Provides nodes for connecting to SGLang Diffusion server and generating images/videos. """ +import base64 +import ipaddress import os +import socket +import urllib.parse import uuid +import comfy.model_management +import comfy.utils import folder_paths import torch @@ -74,6 +80,15 @@ class SGLDH3DistilledSigmas: return (torch.tensor(distilled_h3_sigmas(model_type)),) +def _comfy_job_hooks() -> dict: + """Wire a server-mode job into ComfyUI's progress bar and Cancel button.""" + pbar = comfy.utils.ProgressBar(100) + return { + "progress_callback": pbar.update_absolute, + "check_interrupt": comfy.model_management.throw_exception_if_processing_interrupted, + } + + class SGLDLoraLoader: @classmethod def INPUT_TYPES(cls): @@ -216,7 +231,21 @@ class SGLDiffusionServerModel: "multiline": False, }, ), - } + }, + "optional": { + "image_timeout": ( + "INT", + { + "default": 300, + "min": 1, + "max": 3600, + "tooltip": "Seconds to wait for an image " + "generate/edit request (e.g. a large job, or " + "torch.compile warmup on the first request, can " + "need more than 300s)", + }, + ), + }, } RETURN_TYPES = ("SGLD_CLIENT", "STRING") @@ -224,9 +253,11 @@ class SGLDiffusionServerModel: FUNCTION = "load_server" CATEGORY = "SGLDiffusion" - def load_server(self, base_url: str, api_key: str): + def load_server(self, base_url: str, api_key: str, image_timeout: int = 300): """Initialize OpenAI client for SGLang Diffusion server.""" - client = SGLDiffusionServerAPI(base_url=base_url, api_key=api_key) + client = SGLDiffusionServerAPI( + base_url=base_url, api_key=api_key, image_timeout=image_timeout + ) try: model_info = client.get_model_info() # Format model_info as a readable string @@ -330,11 +361,19 @@ class SGLDiffusionGenerateImage: "one IMAGE batch. Image i uses seed + i.", }, ), + "enhance_prompt": ( + "BOOLEAN", + { + "default": False, + "tooltip": "Rewrite the prompt on the server (needs " + "--prompt-enhancer-config); returned as revised_prompt", + }, + ), }, } - RETURN_TYPES = ("IMAGE",) - RETURN_NAMES = ("image",) + RETURN_TYPES = ("IMAGE", "STRING") + RETURN_NAMES = ("image", "revised_prompt") FUNCTION = "generate_image" CATEGORY = "SGLDiffusion" OUTPUT_NODE = False @@ -352,6 +391,7 @@ class SGLDiffusionGenerateImage: height: int = 1024, enable_teacache: bool = False, num_images: int = 1, + enhance_prompt: bool = False, ): """Generate image using SGLang Diffusion API.""" if not positive_prompt: @@ -382,6 +422,8 @@ class SGLDiffusionGenerateImage: request_params["seed"] = seed if enable_teacache: request_params["enable_teacache"] = True + if enhance_prompt: + request_params["enhance_prompt"] = True if image is not None: # If the image is empty, use the size of the image to generate the image if is_empty_image(image): @@ -397,7 +439,9 @@ class SGLDiffusionGenerateImage: except Exception as e: raise RuntimeError(f"Failed to generate image: {str(e)}") - return (_decode_image_batch(response, num_images),) + images = _decode_image_batch(response, num_images) + revised_prompt = response["data"][0].get("revised_prompt") or positive_prompt + return (images, revised_prompt) class SGLDiffusionGenerateVideo: @@ -431,6 +475,10 @@ class SGLDiffusionGenerateVideo: "tooltip": "input image to use for image-to-video", }, ), + "video": ( + "VIDEO", + {"tooltip": "source clip for video-to-video"}, + ), "seed": ( "INT", { @@ -508,11 +556,19 @@ class SGLDiffusionGenerateVideo: "default": False, }, ), + "enhance_prompt": ( + "BOOLEAN", + { + "default": False, + "tooltip": "Rewrite the prompt on the server (needs " + "--prompt-enhancer-config); returned as revised_prompt", + }, + ), }, } - RETURN_TYPES = ("VIDEO", "STRING") - RETURN_NAMES = ("video", "video_path") + RETURN_TYPES = ("VIDEO", "STRING", "STRING") + RETURN_NAMES = ("video", "video_path", "revised_prompt") FUNCTION = "generate_video" CATEGORY = "SGLDiffusion" OUTPUT_NODE = False @@ -532,6 +588,8 @@ class SGLDiffusionGenerateVideo: fps: int = 24, seconds: int = 5, enable_teacache: bool = False, + video=None, + enhance_prompt: bool = False, ): """Generate video using SGLang Diffusion API.""" if not positive_prompt: @@ -570,18 +628,35 @@ class SGLDiffusionGenerateVideo: request_params["size"] = size else: request_params["input_reference"] = get_image_path(image) + if video is not None: + request_params["video_path"] = os.path.join( + output_dir, f"sgl_input_{uuid.uuid4().hex[:8]}.mp4" + ) + video.save_to(request_params["video_path"]) + if enhance_prompt: + request_params["extra_fields"] = {"enhance_prompt": True} # Call API try: - response = sgld_client.generate_video(**request_params) + response = sgld_client.generate_video( + **request_params, **_comfy_job_hooks() + ) video_path = resolve_video_path( response.get("file_path"), response.get("url") ) + # The server may have resolved a different size than requested + # (e.g. to match an image-to-video input's aspect ratio). + resolved_size = response.get("size", "") + if resolved_size: + width, height = (int(v) for v in resolved_size.split("x")) video = convert_video_to_comfy_video(video_path, height, width) + except comfy.model_management.InterruptProcessingException: + raise except Exception as e: raise RuntimeError(f"Failed to generate video: {str(e)}") - return (video, video_path) + revised_prompt = response.get("revised_prompt") or positive_prompt + return (video, video_path, revised_prompt) class SGLDiffusionGenerateH3: @@ -668,11 +743,61 @@ class SGLDiffusionGenerateH3: OUTPUT_NODE = False @staticmethod - def _material_uri(value: str) -> str: - """Local paths become file:// URIs; remote URLs are passed through.""" + def _image_material_uri(image: torch.Tensor) -> str: + """Embed the image inline as a base64 data URI. + + A `file://` path only works when the SGLD server runs on this same + machine; the server's H3 material loader accepts `data:` URIs + directly (no upload endpoint needed), so this works locally and + remotely alike. + """ + path = get_image_path(image) + try: + with open(path, "rb") as fh: + encoded = base64.b64encode(fh.read()).decode("ascii") + finally: + os.remove(path) + return f"data:image/png;base64,{encoded}" + + @staticmethod + def _is_local_server(base_url: str) -> bool: + host = (urllib.parse.urlparse(base_url).hostname or "").lower() + if not host: + return False + if host in ("localhost", "0.0.0.0", "::"): + return True + own = {socket.gethostname().lower(), socket.getfqdn().lower()} + if host in own: + return True + try: + infos = socket.getaddrinfo(host, None) + except (OSError, UnicodeError): + return False + addrs = {info[4][0].split("%", 1)[0] for info in infos} + return bool(addrs) and all( + ipaddress.ip_address(addr).is_loopback for addr in addrs + ) + + @classmethod + def _remote_material_uri(cls, value: str, kind: str, base_url: str) -> str: + """Remote URLs pass through. A local path only works when the SGLD + server is also on this machine: H3's server-side `/v1/videos` + endpoint has no multipart slot for the video/audio conditions this + node sends through `extra_fields.conditions`, unlike the image + conditions above (which sidestep that by embedding bytes inline). + Raise clearly instead of building a `file://` URI that will + silently fail against any server that isn't local. + """ + # file:// already names a server-side path if value.startswith(("http://", "https://", "file://")): return value - return f"file://{os.path.abspath(value)}" + if cls._is_local_server(base_url): + return f"file://{os.path.abspath(value)}" + raise ValueError( + f"{kind} must be an http(s) URL when the SGLD server " + f"({base_url}) isn't on this machine; got a local path " + f"({value!r}), which that server has no way to read or upload." + ) def generate( self, @@ -705,7 +830,7 @@ class SGLDiffusionGenerateH3: conditions.append( { "type": "image", - "uri": self._material_uri(get_image_path(first_frame)), + "uri": self._image_material_uri(first_frame), "role": "keyframe", "frame_index": 0, } @@ -714,7 +839,7 @@ class SGLDiffusionGenerateH3: conditions.append( { "type": "image", - "uri": self._material_uri(get_image_path(last_frame)), + "uri": self._image_material_uri(last_frame), "role": "keyframe", "frame_index": -1, } @@ -723,7 +848,7 @@ class SGLDiffusionGenerateH3: conditions.append( { "type": "image", - "uri": self._material_uri(get_image_path(reference_image)), + "uri": self._image_material_uri(reference_image), "role": "reference", } ) @@ -731,7 +856,9 @@ class SGLDiffusionGenerateH3: conditions.append( { "type": "video", - "uri": self._material_uri(reference_video), + "uri": self._remote_material_uri( + reference_video, "reference_video", sgld_client.base_url + ), "role": "reference", } ) @@ -739,7 +866,9 @@ class SGLDiffusionGenerateH3: conditions.append( { "type": "audio", - "uri": self._material_uri(reference_audio), + "uri": self._remote_material_uri( + reference_audio, "reference_audio", sgld_client.base_url + ), "role": "reference", } ) @@ -783,10 +912,14 @@ class SGLDiffusionGenerateH3: request_params["seed"] = seed try: - response = sgld_client.generate_video(**request_params) + response = sgld_client.generate_video( + **request_params, **_comfy_job_hooks() + ) video_path = resolve_video_path( response.get("file_path"), response.get("url") ) + except comfy.model_management.InterruptProcessingException: + raise except Exception as e: raise RuntimeError(f"Failed to generate MiniMax-H3 video: {str(e)}") @@ -838,6 +971,17 @@ class SGLDiffusionServerSetLora: "tooltip": "Which transformer(s) to apply the LoRA to", }, ), + "strength": ( + "FLOAT", + { + "default": 1.0, + "min": 0.0, + "max": 10.0, + "step": 0.01, + "tooltip": "LoRA merge strength; the server's own " + "default is 1.0", + }, + ), }, } @@ -853,6 +997,7 @@ class SGLDiffusionServerSetLora: lora_name: str = "", lora_nickname: str = "", target: str = "all", + strength: float = 1.0, ): """Set LoRA adapter using SGLang Diffusion API.""" if lora_nickname == "": @@ -863,6 +1008,7 @@ class SGLDiffusionServerSetLora: "lora_nickname": lora_nickname, "lora_path": lora_name, "target": target, + "strength": strength, } # Call API diff --git a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/test/test_h3_request.py b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/test/test_h3_request.py index 6b5b8e231f..69b2ee0861 100644 --- a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/test/test_h3_request.py +++ b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/test/test_h3_request.py @@ -5,7 +5,9 @@ only HTTP, so a change on either side of the payload contract fails here. """ import importlib.util +import os import sys +import tempfile import types from pathlib import Path from unittest import mock @@ -125,25 +127,56 @@ class _Response: return self._payload +_TMP_FILES = [] + + +def _tmp_file(suffix): + """A temp file path that the autouse fixture below deletes afterwards.""" + with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as fh: + _TMP_FILES.append(fh.name) + return fh.name + + +@pytest.fixture(autouse=True) +def _remove_tmp_files(): + yield + while _TMP_FILES: + path = _TMP_FILES.pop() + if os.path.exists(path): + os.remove(path) + + def _run_node(**node_kwargs): """Drive the node through the real client and capture the POST payload.""" client = SGLDiffusionServerAPI(base_url="http://127.0.0.1:30010") captured = {} + # A real, existing path: the client now checks the server's reported + # file_path with os.path.exists before trusting it (see + # SGLDiffusionServerAPI._localize_video_result), since on a remote + # server that path belongs to a different filesystem. + out_path = _tmp_file(".mp4") - def fake_post(url, json=None, headers=None, timeout=None): - captured.update(json) + def fake_post(url, json=None, headers=None, timeout=None, files=None, data=None): + captured.update(json or data or {}) return _Response({"id": "job-1"}) - def fake_get(url, headers=None, timeout=None): + def fake_get(url, headers=None, timeout=None, **kwargs): return _Response( { "id": "job-1", "status": "completed", "size": RESOLVED_SIZE, - "file_path": "/tmp/out.mp4", + "file_path": out_path, } ) + def fake_get_image_path(image): + # _image_material_uri reads this file's bytes and then deletes it. + path = _tmp_file(".png") + with open(path, "wb") as fh: + fh.write(b"\x89PNG\r\n\x1a\n" + b"\x00" * 16) + return path + node = SGLDiffusionGenerateH3() with ( mock.patch( @@ -156,7 +189,7 @@ def _run_node(**node_kwargs): ), mock.patch( f"{PKG}.nodes.get_image_path", - side_effect=lambda image: "/tmp/frame.png", + side_effect=fake_get_image_path, ), ): result = node.generate(sgld_client=client, **node_kwargs) @@ -191,7 +224,11 @@ def test_fl2va_maps_keyframes_to_frame_indices(): assert [c["role"] for c in payload["conditions"]] == ["keyframe", "keyframe"] assert [c["frame_index"] for c in payload["conditions"]] == [0, -1] - assert all(c["uri"].startswith("file:///") for c in payload["conditions"]) + # Images are embedded inline (data: URI), not as a local file:// path + # the server may not be able to read. + assert all( + c["uri"].startswith("data:image/png;base64,") for c in payload["conditions"] + ) def test_ref2va_preserves_modality_order_for_prompt_tags(): @@ -222,7 +259,7 @@ def test_node_reports_the_server_resolved_canvas(): _, (video, video_path) = _run_node(positive_prompt="a cat", task="t2va") assert video.get_dimensions() == (1344, 768) - assert video_path == "/tmp/out.mp4" + assert os.path.exists(video_path) @pytest.mark.parametrize( @@ -242,9 +279,10 @@ def test_extra_fields_win_over_generic_defaults(): """A model's own field must not be shadowed by a same-named generic default.""" client = SGLDiffusionServerAPI(base_url="http://127.0.0.1:30010") captured = {} + out_path = _tmp_file(".mp4") - def fake_post(url, json=None, headers=None, timeout=None): - captured.update(json) + def fake_post(url, json=None, headers=None, timeout=None, files=None, data=None): + captured.update(json or data or {}) return _Response({"id": "job-1"}) with ( @@ -255,7 +293,12 @@ def test_extra_fields_win_over_generic_defaults(): mock.patch( f"{PKG}.core.server_api.requests.get", side_effect=lambda *a, **k: _Response( - {"id": "job-1", "status": "completed", "size": RESOLVED_SIZE} + { + "id": "job-1", + "status": "completed", + "size": RESOLVED_SIZE, + "file_path": out_path, + } ), ), ): diff --git a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/test/test_sgld_server_mode.py b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/test/test_sgld_server_mode.py new file mode 100644 index 0000000000..a9f1176c8f --- /dev/null +++ b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/test/test_sgld_server_mode.py @@ -0,0 +1,758 @@ +"""Regression tests for ComfyUI_SGLDiffusion server-mode bugs found in audit. + +Each bug gets a test that fails against the pre-fix code and passes after. +These stub ComfyUI/ torch-adjacent modules the same way test_h3_request.py +does, so they run without a ComfyUI install or a GPU. +""" + +import base64 +import importlib.util +import io +import os +import sys +import tempfile +import types +from pathlib import Path +from unittest import mock + +import pytest +import torch +from PIL import Image + +PLUGIN_DIR = Path(__file__).resolve().parents[1] +PKG = "sgld_server_mode_under_test" + + +def _install_comfy_stubs() -> None: + folder_paths = types.ModuleType("folder_paths") + _tmp = tempfile.mkdtemp(prefix="sgld_test_temp_") + folder_paths.get_temp_directory = lambda: _tmp + folder_paths._diffusion_models_dirs = [] + folder_paths.folder_names_and_paths = { + "diffusion_models": ([], {".safetensors", ".sft"}), + "loras": ([], {".safetensors"}), + } + + def get_filename_list(name): + names = set() + for folder in folder_paths.folder_names_and_paths.get(name, ([], set()))[0]: + if os.path.isdir(folder): + names.update(os.listdir(folder)) + return sorted(names) + + def get_folder_paths(name): + return list(folder_paths.folder_names_and_paths.get(name, ([], set()))[0]) + + def get_full_path(name, filename): + for folder in get_folder_paths(name): + candidate = os.path.join(folder, filename) + if os.path.isfile(candidate): + return candidate + return None + + folder_paths.get_filename_list = get_filename_list + folder_paths.get_folder_paths = get_folder_paths + folder_paths.get_full_path = get_full_path + sys.modules["folder_paths"] = folder_paths + + comfy_api = types.ModuleType("comfy_api") + comfy_api_input = types.ModuleType("comfy_api.input") + + class VideoInput: + pass + + comfy_api_input.VideoInput = VideoInput + comfy_api.input = comfy_api_input + sys.modules["comfy_api"] = comfy_api + sys.modules["comfy_api.input"] = comfy_api_input + # No comfy_api.input_impl: exercises the SGLDVideoInput fallback path. + sys.modules.pop("comfy_api.input_impl", None) + + comfy = types.ModuleType("comfy") + comfy.model_detection = types.ModuleType("comfy.model_detection") + comfy.model_management = types.ModuleType("comfy.model_management") + + comfy_utils = types.ModuleType("comfy.utils") + for name in ( + "calculate_parameters", + "load_torch_file", + "state_dict_prefix_replace", + "unet_to_diffusers", + ): + setattr(comfy_utils, name, lambda *a, **k: None) + + class InterruptProcessingException(Exception): + pass + + _interrupt = {"flag": False} + + def throw_exception_if_processing_interrupted(): + if _interrupt["flag"]: + raise InterruptProcessingException() + + comfy.model_management.InterruptProcessingException = InterruptProcessingException + comfy.model_management.throw_exception_if_processing_interrupted = ( + throw_exception_if_processing_interrupted + ) + comfy.model_management._test_interrupt = _interrupt + + class ProgressBar: + instances = [] + + def __init__(self, total): + self.total = total + self.values = [] + ProgressBar.instances.append(self) + + def update_absolute(self, value, total=None, preview=None): + self.values.append(value) + + comfy_utils.ProgressBar = ProgressBar + comfy.utils = comfy_utils + + comfy_model_patcher = types.ModuleType("comfy.model_patcher") + + class ModelPatcher: + def __init__(self, model, load_device, offload_device, size=0, *a, **k): + self.model = model + self.load_device = load_device + self.offload_device = offload_device + self.size = size + self.patches = {} + self.object_patches = {} + self.model_options = {} + self.backup = {} + self.object_patches_backup = {} + self.wrappers = {} + self.callbacks = {} + self.patches_uuid = None + self.weight_inplace_update = False + + def add_wrapper_with_key(self, *a, **k): + pass + + # ComfyUI: loaded_size() reads the model's own weight counter (0 for + # SGLD, whose weights live in the worker); detach() is what + # LoadedModel.model_unload always ends in. + def loaded_size(self): + return getattr(self.model, "model_loaded_weight_memory", 0) + + def detach(self, unpatch_all=True): + return self.model + + comfy_model_patcher.ModelPatcher = ModelPatcher + comfy.model_patcher = comfy_model_patcher + + comfy_patcher_extension = types.ModuleType("comfy.patcher_extension") + + class WrappersMP: + SAMPLER_SAMPLE = "SAMPLER_SAMPLE" + + comfy_patcher_extension.WrappersMP = WrappersMP + comfy.patcher_extension = comfy_patcher_extension + + sys.modules["comfy"] = comfy + sys.modules["comfy.model_detection"] = comfy.model_detection + sys.modules["comfy.model_management"] = comfy.model_management + sys.modules["comfy.utils"] = comfy_utils + sys.modules["comfy.model_patcher"] = comfy_model_patcher + sys.modules["comfy.patcher_extension"] = comfy_patcher_extension + return folder_paths + + +def _load(module_name: str, relative_path: str): + spec = importlib.util.spec_from_file_location( + module_name, PLUGIN_DIR / relative_path + ) + module = importlib.util.module_from_spec(spec) + sys.modules[module_name] = module + spec.loader.exec_module(module) + return module + + +def _load_plugin(): + folder_paths = _install_comfy_stubs() + + package = types.ModuleType(PKG) + package.__path__ = [str(PLUGIN_DIR)] + sys.modules[PKG] = package + + core = types.ModuleType(f"{PKG}.core") + core.__path__ = [str(PLUGIN_DIR / "core")] + sys.modules[f"{PKG}.core"] = core + + server_api = _load(f"{PKG}.core.server_api", "core/server_api.py") + core.SGLDiffusionServerAPI = server_api.SGLDiffusionServerAPI + model_patcher = _load(f"{PKG}.core.model_patcher", "core/model_patcher.py") + core.SGLDModelPatcher = model_patcher.SGLDModelPatcher + generator = _load(f"{PKG}.core.generator", "core/generator.py") + core.SGLDiffusionGenerator = generator.SGLDiffusionGenerator + + _load(f"{PKG}.utils", "utils.py") + nodes = _load(f"{PKG}.nodes", "nodes.py") + return ( + server_api, + model_patcher, + generator, + nodes, + sys.modules[f"{PKG}.utils"], + folder_paths, + ) + + +SERVER_API, MODEL_PATCHER, GENERATOR, NODES, UTILS, FOLDER_PATHS = _load_plugin() +SGLDiffusionServerAPI = SERVER_API.SGLDiffusionServerAPI +SGLDModelPatcher = MODEL_PATCHER.SGLDModelPatcher + + +class _Response: + def __init__(self, payload, status_code=200): + self._payload = payload + self.status_code = status_code + + def raise_for_status(self): + pass + + def json(self): + return self._payload + + def iter_content(self, chunk_size=None): + yield b"fake-mp4-bytes" + + +# --- Bug 9: hardcoded 300s timeout on image requests ----------------------- + + +_TMP_FILES = [] + + +def _tmp_file(suffix): + """A temp file path that the autouse fixture below deletes afterwards.""" + with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as fh: + _TMP_FILES.append(fh.name) + return fh.name + + +@pytest.fixture(autouse=True) +def _remove_tmp_files(): + yield + while _TMP_FILES: + path = _TMP_FILES.pop() + if os.path.exists(path): + os.remove(path) + + +def test_image_timeout_is_configurable(): + client = SGLDiffusionServerAPI(base_url="http://127.0.0.1:1234", image_timeout=900) + captured = {} + + def fake_post(url, json=None, headers=None, timeout=None, **kwargs): + captured["timeout"] = timeout + return _Response({"data": [{"b64_json": "x"}]}) + + with mock.patch(f"{PKG}.core.server_api.requests.post", side_effect=fake_post): + client.generate_image(prompt="a cat") + + assert captured["timeout"] == 900 + + +# --- Bug 10: batched image inputs were silently truncated to image[0] ------ + + +def test_get_image_path_uses_first_image_of_a_batch_with_a_warning(caplog): + batch = torch.zeros(3, 8, 8, 3) + with caplog.at_level("WARNING"): + path = UTILS.get_image_path(batch) + assert os.path.exists(path) + os.remove(path) + assert "batch of 3 images" in caplog.text + + +def test_get_image_path_accepts_single_image(): + single = torch.zeros(1, 8, 8, 3) + path = UTILS.get_image_path(single) + assert os.path.exists(path) + os.remove(path) + + +# --- Bug 11: LoRA strength could not be set in server mode ----------------- + + +def test_set_lora_sends_strength_to_server(): + client = SGLDiffusionServerAPI(base_url="http://127.0.0.1:1234") + captured = {} + + def fake_post(url, json=None, headers=None, timeout=None, **kwargs): + captured.update(json) + return _Response({"status": "ok"}) + + with mock.patch(f"{PKG}.core.server_api.requests.post", side_effect=fake_post): + client.set_lora( + lora_nickname="style", lora_path="style.safetensors", strength=0.7 + ) + + assert captured["strength"] == 0.7 + + +def test_set_lora_node_exposes_strength(monkeypatch): + client = SGLDiffusionServerAPI(base_url="http://127.0.0.1:1234") + node = NODES.SGLDiffusionServerSetLora() + captured = {} + + def fake_set_lora(**kwargs): + captured.update(kwargs) + return {"status": "ok"} + + monkeypatch.setattr(client, "set_lora", fake_set_lora) + node.set_lora(sgld_client=client, lora_name="style.safetensors", strength=0.5) + assert captured["strength"] == 0.5 + + +# --- Bug 6/7: SGLDVideoInput didn't implement VideoInput; dims could be wrong + + +def test_video_input_fallback_fails_loudly_instead_of_returning_wrong_data(): + video = UTILS.convert_video_to_comfy_video("/tmp/out.mp4", height=720, width=1280) + assert isinstance(video, UTILS.SGLDVideoInput) # no comfy_api.input_impl stubbed + with pytest.raises(NotImplementedError): + video.get_components() + + +def test_video_input_save_to_refuses_silent_format_drop(): + video = UTILS.SGLDVideoInput("/tmp/out.mp4", height=720, width=1280) + with pytest.raises(NotImplementedError): + video.save_to("/tmp/converted.webm", format="webm") + + +def test_generic_video_node_reports_server_resolved_size_not_requested_size(): + client = SGLDiffusionServerAPI(base_url="http://127.0.0.1:1234") + node = NODES.SGLDiffusionGenerateVideo() + + with ( + mock.patch.object( + client, + "generate_video", + return_value={"file_path": "/tmp/x.mp4", "size": "1280x704"}, + ), + ): + video, _, _ = node.generate_video( + sgld_client=client, + positive_prompt="a cat", + width=1280, + height=720, # requested 720, server actually resolved to 704 + ) + assert video.get_dimensions() == (1280, 704) + + +# --- Bug 3: server-mode nodes sent local paths a remote server can't read -- + + +def test_generate_video_uploads_local_input_reference_instead_of_sending_path(): + client = SGLDiffusionServerAPI(base_url="http://127.0.0.1:1234") + local_image = _tmp_file(".png") + with open(local_image, "wb") as fh: + fh.write(b"fake-png-bytes") + + seen = {} + + def fake_post(url, json=None, headers=None, timeout=None, files=None, data=None): + seen["files"] = files + seen["data"] = data + seen["json"] = json + return _Response({"id": "job-1"}) + + out_path = _tmp_file(".mp4") + + def fake_get(url, headers=None, timeout=None, **kwargs): + return _Response({"id": "job-1", "status": "completed", "file_path": out_path}) + + with ( + mock.patch(f"{PKG}.core.server_api.requests.post", side_effect=fake_post), + mock.patch(f"{PKG}.core.server_api.requests.get", side_effect=fake_get), + ): + client.generate_video(prompt="a cat", input_reference=local_image) + + os.remove(local_image) + # A local path must be uploaded as file bytes, not sent as a path string + # the server (possibly on another machine) has no way to read. + assert seen["files"] is not None + assert seen["json"] is None + + +def _capture_video_request(client, **kwargs): + seen = {} + + def fake_post(url, json=None, headers=None, timeout=None, files=None, data=None): + seen["files"] = files and { + key: (name, fh.read(), content_type) + for key, (name, fh, content_type) in files.items() + } + seen["data"] = data + seen["json"] = json + return _Response({"id": "job-1"}) + + out_path = _tmp_file(".mp4") + + def fake_get(url, headers=None, timeout=None, **kw): + return _Response({"id": "job-1", "status": "completed", "file_path": out_path}) + + with ( + mock.patch(f"{PKG}.core.server_api.requests.post", side_effect=fake_post), + mock.patch(f"{PKG}.core.server_api.requests.get", side_effect=fake_get), + ): + client.generate_video(prompt="a cat", **kwargs) + return seen + + +def test_generate_video_uploads_local_source_video_as_video_reference(): + client = SGLDiffusionServerAPI(base_url="http://127.0.0.1:1234") + with tempfile.TemporaryDirectory() as tmp: + clip = os.path.join(tmp, "clip.mp4") + image = os.path.join(tmp, "ref.png") + for path, body in ((clip, b"fake-mp4"), (image, b"fake-png")): + with open(path, "wb") as fh: + fh.write(body) + + seen = _capture_video_request(client, video_path=clip, input_reference=image) + + # Both local files go up as multipart; neither path string is sent. + assert seen["json"] is None + assert seen["files"]["video_reference"] == ("clip.mp4", b"fake-mp4", "video/mp4") + assert seen["files"]["input_reference"][1] == b"fake-png" + assert "video_path" not in seen["data"] + assert "input_reference" not in seen["data"] + + +def test_generate_video_multipart_fields_are_json_encoded(): + client = SGLDiffusionServerAPI(base_url="http://127.0.0.1:1234") + with tempfile.TemporaryDirectory() as tmp: + image = os.path.join(tmp, "ref.png") + with open(image, "wb") as fh: + fh.write(b"fake-png") + seen = _capture_video_request( + client, + input_reference=image, + extra_fields={"target": {"type": "image"}, "flag": False}, + ) + + # the server json-decodes form values, so Python repr would break them + assert seen["data"]["target"] == '{"type": "image"}' + assert seen["data"]["flag"] == "false" + assert seen["data"]["prompt"] == "a cat" + + +def test_generate_video_sends_remote_source_video_as_video_path(): + client = SGLDiffusionServerAPI(base_url="http://127.0.0.1:1234") + for source in ("https://example.com/clip.mp4", "/srv/sglang/inputs/clip.mp4"): + seen = _capture_video_request(client, video_path=source) + # A URL or a path that only exists on the server stays a string. + assert seen["files"] is None + assert seen["json"]["video_path"] == source + + +def test_generate_video_downloads_result_when_server_path_is_not_local(): + client = SGLDiffusionServerAPI(base_url="http://127.0.0.1:1234") + + def fake_post(url, **kwargs): + return _Response({"id": "job-1"}) + + def fake_get(url, headers=None, timeout=None, stream=False, **kwargs): + if url.endswith("/content"): + return _Response({}, status_code=200) + return _Response( + { + "id": "job-1", + "status": "completed", + # A path on the *server's* filesystem, not this one. + "file_path": "/srv/sglang/outputs/job-1.mp4", + } + ) + + with tempfile.TemporaryDirectory() as out_dir: + with ( + mock.patch(f"{PKG}.core.server_api.requests.post", side_effect=fake_post), + mock.patch(f"{PKG}.core.server_api.requests.get", side_effect=fake_get), + ): + result = client.generate_video(prompt="a cat", output_path=out_dir) + + assert os.path.exists(result["file_path"]) + assert os.path.realpath( + os.path.dirname(result["file_path"]) + ) == os.path.realpath(out_dir) + + +@pytest.mark.parametrize("video_id", ["../escape", "a/b", "job.1", ""]) +def test_localize_video_result_rejects_unsafe_video_id(video_id): + client = SGLDiffusionServerAPI(base_url="http://127.0.0.1:1234") + status = {"status": "completed", "file_path": "/srv/out/x.mp4"} + with tempfile.TemporaryDirectory() as out_dir: + with mock.patch(f"{PKG}.core.server_api.requests.get") as get: + with pytest.raises(RuntimeError, match="unsafe id"): + client._localize_video_result(status, video_id, out_dir) + get.assert_not_called() + assert os.listdir(out_dir) == [] + + +def test_localize_video_result_whitelists_extension_and_cleans_up_on_error(): + client = SGLDiffusionServerAPI(base_url="http://127.0.0.1:1234") + + class _Broken(_Response): + def iter_content(self, chunk_size=None): + yield b"partial" + raise SERVER_API.requests.exceptions.ConnectionError("dropped") + + status = {"status": "completed", "file_path": "/srv/out/job-1.sh"} + with tempfile.TemporaryDirectory() as out_dir: + with mock.patch( + f"{PKG}.core.server_api.requests.get", return_value=_Response({}) + ): + result = client._localize_video_result(status, "job-1", out_dir) + assert result["file_path"] == os.path.join( + os.path.realpath(out_dir), "job-1.mp4" + ) + os.remove(result["file_path"]) + with mock.patch( + f"{PKG}.core.server_api.requests.get", return_value=_Broken({}) + ): + with pytest.raises(SERVER_API.requests.exceptions.ConnectionError): + client._localize_video_result(status, "job-1", out_dir) + assert os.listdir(out_dir) == [] + + +class _HTTPErrorResponse(_Response): + def raise_for_status(self): + raise SERVER_API.requests.exceptions.HTTPError( + f"{self.status_code} Client Error", response=self + ) + + +def _completed_video_client(job, max_polls=3): + """Client whose job is already completed; /content answers 400, as the + server does for a job uploaded to cloud storage. Polling past + `max_polls` fails the test instead of looping until max_wait_time.""" + client = SGLDiffusionServerAPI(base_url="http://127.0.0.1:1234") + calls = {"status": 0, "content": 0} + + def fake_get(url, **kwargs): + if url.endswith("/content"): + calls["content"] += 1 + return _HTTPErrorResponse({"detail": "use the cloud URL"}, 400) + calls["status"] += 1 + assert calls["status"] <= max_polls, "kept polling a completed job" + return _Response(job) + + def fake_post(url, **kwargs): + return _Response({"id": "job-1"}) + + patches = ( + mock.patch.object(SERVER_API.requests, "get", fake_get), + mock.patch.object(SERVER_API.requests, "post", fake_post), + mock.patch.object(SERVER_API.time, "sleep"), + ) + return client, calls, patches + + +def test_generate_video_returns_cloud_url_without_downloading_content(): + url = "https://bucket.example/job-1.mp4" + client, calls, patches = _completed_video_client( + {"id": "job-1", "status": "completed", "url": url, "file_path": None} + ) + with tempfile.TemporaryDirectory() as out_dir: + with patches[0], patches[1], patches[2]: + result = client.generate_video(prompt="a cat", output_path=out_dir) + # resolve_video_path fetches the url, as on main. + assert result["url"] == url + assert result["file_path"] is None + assert calls["content"] == 0 + + +def test_generate_video_fails_fast_when_result_download_fails(): + client, calls, patches = _completed_video_client( + {"id": "job-1", "status": "completed", "file_path": "/srv/out/job-1.mp4"} + ) + with tempfile.TemporaryDirectory() as out_dir: + with patches[0], patches[1], patches[2]: + with pytest.raises(RuntimeError, match="400"): + client.generate_video(prompt="a cat", output_path=out_dir) + assert calls == {"status": 1, "content": 1} + + +def test_h3_image_conditions_use_data_uri_not_local_file_path(): + node = NODES.SGLDiffusionGenerateH3() + image = torch.zeros(1, 8, 8, 3) + uri = node._image_material_uri(image) + assert uri.startswith("data:image/png;base64,") + + +def test_h3_rejects_local_video_reference_against_remote_server(): + with pytest.raises(ValueError, match="no way to read or upload"): + NODES.SGLDiffusionGenerateH3._remote_material_uri( + "/data/clip.mp4", "reference_video", "http://remote-host.example:3000/v1" + ) + + +def test_h3_allows_local_video_reference_against_local_server(): + uri = NODES.SGLDiffusionGenerateH3._remote_material_uri( + "/data/clip.mp4", "reference_video", "http://127.0.0.1:3000/v1" + ) + assert uri == "file:///data/clip.mp4" + + +def test_is_local_server_accepts_own_hostname_and_loopback(): + is_local = NODES.SGLDiffusionGenerateH3._is_local_server + host = NODES.socket.gethostname() + assert is_local(f"http://{host}:3000/v1") + assert is_local("http://0.0.0.0:3000/v1") + assert is_local("http://127.0.0.2:3000/v1") + assert is_local("http://[::1]:3000/v1") + with mock.patch.object( + NODES.socket, "getaddrinfo", return_value=[(0, 0, 0, "", ("10.1.2.3", 0))] + ): + assert not is_local("http://gpu-box:3000/v1") + + +def test_h3_passes_file_uri_through_for_any_server(): + # a file:// URI names a path on the server, so it is never rewritten + for base_url in ("http://127.0.0.1:3000/v1", "http://remote-host.example:3000/v1"): + uri = NODES.SGLDiffusionGenerateH3._remote_material_uri( + "file:///srv/clip.mp4", "reference_video", base_url + ) + assert uri == "file:///srv/clip.mp4" + + +# --- server-mode video jobs: no progress bar, Cancel was ignored ------------ + + +def _video_client_with_statuses(statuses): + client = SGLDiffusionServerAPI(base_url="http://127.0.0.1:1234") + polled = iter(statuses) + + def fake_get(url, **kwargs): + return _Response(next(polled)) + + def fake_post(url, **kwargs): + return _Response({"id": "vid1"}) + + return client, fake_get, fake_post + + +def test_generate_video_reports_server_progress(): + client, fake_get, fake_post = _video_client_with_statuses( + [ + {"status": "queued", "progress": 0}, + {"status": "in_progress", "progress": 40}, + {"status": "completed", "progress": 100, "file_path": "/tmp/x.mp4"}, + ] + ) + seen = [] + with ( + mock.patch.object(SERVER_API.requests, "get", fake_get), + mock.patch.object(SERVER_API.requests, "post", fake_post), + mock.patch.object(SERVER_API.time, "sleep"), + mock.patch.object(client, "_localize_video_result", lambda s, *a: s), + ): + client.generate_video(prompt="a cat", progress_callback=seen.append) + assert seen == [0, 40, 100] + + +def test_generate_video_stops_polling_when_interrupted(): + client, fake_get, fake_post = _video_client_with_statuses( + [{"status": "in_progress", "progress": 10}] * 50 + ) + checks = {"n": 0} + + class Interrupted(Exception): + pass + + def check_interrupt(): + checks["n"] += 1 + if checks["n"] > 2: + raise Interrupted() + + with ( + mock.patch.object(SERVER_API.requests, "get", fake_get), + mock.patch.object(SERVER_API.requests, "post", fake_post), + mock.patch.object(SERVER_API.time, "sleep"), + ): + with pytest.raises(Interrupted): + client.generate_video(prompt="a cat", check_interrupt=check_interrupt) + assert checks["n"] == 3 # stopped promptly, did not poll to completion + + +def test_video_node_wires_comfy_progress_bar_and_interrupt(): + client = SGLDiffusionServerAPI(base_url="http://127.0.0.1:1234") + node = NODES.SGLDiffusionGenerateVideo() + captured = {} + + def fake_generate_video(**kwargs): + captured.update(kwargs) + return {"file_path": "/tmp/x.mp4", "size": "1280x720"} + + with mock.patch.object(client, "generate_video", fake_generate_video): + node.generate_video( + sgld_client=client, positive_prompt="a cat", width=1280, height=720 + ) + captured["progress_callback"](55) + assert sys.modules["comfy.utils"].ProgressBar.instances[-1].values == [55] + sys.modules["comfy.model_management"]._test_interrupt["flag"] = True + try: + with pytest.raises( + sys.modules["comfy.model_management"].InterruptProcessingException + ): + captured["check_interrupt"]() + finally: + sys.modules["comfy.model_management"]._test_interrupt["flag"] = False + + +@pytest.mark.parametrize("enhance", [True, False]) +def test_image_node_enhance_prompt_returns_revised_prompt(enhance): + client = SGLDiffusionServerAPI(base_url="http://127.0.0.1:1234") + png = io.BytesIO() + Image.new("RGB", (8, 8)).save(png, format="PNG") + b64 = base64.b64encode(png.getvalue()).decode() + revised = "a fluffy cat" if enhance else None + captured = {} + + def fake_post(url, json=None, **kwargs): + captured.update(json) + return _Response({"data": [{"b64_json": b64, "revised_prompt": revised}]}) + + with mock.patch(f"{PKG}.core.server_api.requests.post", side_effect=fake_post): + image, revised_prompt = NODES.SGLDiffusionGenerateImage().generate_image( + sgld_client=client, positive_prompt="a cat", enhance_prompt=enhance + ) + + assert captured.get("enhance_prompt", False) == enhance + assert image.shape == (1, 8, 8, 3) + assert revised_prompt == ("a fluffy cat" if enhance else "a cat") + + +def test_video_node_sends_source_video_and_enhance_prompt(): + client = SGLDiffusionServerAPI(base_url="http://127.0.0.1:1234") + captured = {} + + class SourceVideo: + def save_to(self, path): + with open(path, "wb") as fh: + fh.write(b"source-mp4") + + def fake_generate_video(**kwargs): + captured.update(kwargs) + with open(kwargs["video_path"], "rb") as fh: + captured["video_bytes"] = fh.read() + return {"file_path": "/tmp/x.mp4", "revised_prompt": "a fluffy cat"} + + with mock.patch.object(client, "generate_video", fake_generate_video): + _, video_path, revised_prompt = ( + NODES.SGLDiffusionGenerateVideo().generate_video( + sgld_client=client, + positive_prompt="a cat", + video=SourceVideo(), + enhance_prompt=True, + ) + ) + os.remove(captured["video_path"]) + + assert captured["video_bytes"] == b"source-mp4" + assert captured["extra_fields"] == {"enhance_prompt": True} + assert (video_path, revised_prompt) == ("/tmp/x.mp4", "a fluffy cat") diff --git a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/utils.py b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/utils.py index 6d7da22501..cb2ac75cc6 100644 --- a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/utils.py +++ b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/utils.py @@ -1,5 +1,6 @@ import base64 import io +import logging import os import shutil import subprocess @@ -13,6 +14,8 @@ import torch from comfy_api.input import VideoInput from PIL import Image +logger = logging.getLogger(__name__) + def _ensure_dir(path: str) -> None: os.makedirs(path, exist_ok=True) @@ -82,7 +85,16 @@ def is_empty_image(image: torch.Tensor, tolerance: float = 1e-6) -> bool: def get_image_path(image: torch.Tensor) -> str: """ Save tensor image to ComfyUI temp directory as PNG and return the path. + + This input only has a single image slot, so a batch keeps its first + image (as before) and logs a warning that the rest are dropped. """ + if image.dim() == 4 and image.shape[0] > 1: + logger.warning( + "get_image_path received a batch of %d images but this input only " + "accepts one; using the first. Use 'Image From Batch' to pick another.", + image.shape[0], + ) temp_dir = folder_paths.get_temp_directory() # Build file name @@ -131,6 +143,14 @@ def convert_b64_to_tensor_image(b64_image: str) -> torch.Tensor: class SGLDVideoInput(VideoInput): + """Minimal fallback used only when ComfyUI's own file-backed VideoInput + isn't importable. It cannot decode frames/audio or re-encode on save, so + it fails loudly on those rather than silently returning wrong data (the + previous ``get_components`` returned ``[video_path]`` instead of a + ``VideoComponents``, and ``save_to`` dropped ``format``/``codec``/ + ``metadata`` without telling the caller). + """ + def __init__(self, video_path: str, height: int, width: int): super().__init__() @@ -148,16 +168,19 @@ class SGLDVideoInput(VideoInput): return self.width, self.height def get_components(self): - """ - Returns the components of the video input. - This is required by the VideoInput abstract base class. - """ - return [self.video_path] + raise NotImplementedError( + "SGLDVideoInput cannot decode frames/audio (no ComfyUI " + "VideoFromFile available in this environment); pass the file " + "at video_path to a node that reads files directly instead." + ) def save_to(self, path: str, format=None, codec=None, metadata=None): - """ - Abstract method to save the video input to a file. - """ + if format is not None or codec is not None or metadata is not None: + raise NotImplementedError( + "SGLDVideoInput can only copy the file as-is; format/codec/" + "metadata conversion needs ComfyUI's own VideoFromFile, " + "which is unavailable in this environment." + ) if not os.path.exists(self.video_path): raise FileNotFoundError( f"Video not found at '{self.video_path}'. This file must be " @@ -236,7 +259,18 @@ def convert_video_to_comfy_video( video_path: str, height: int, width: int ) -> VideoInput: """ - Convert video to ComfyUI VIDEO format (VideoInput). + Convert a video file on disk to ComfyUI's VIDEO type. + + Prefers ComfyUI's own file-backed ``VideoFromFile``: it probes the real + file for dimensions/frames/audio, so it is also immune to the + width/height we were told (which can be wrong, e.g. when the server + picked a different size than requested to match an image-to-video + input). ``height``/``width`` are only used by the ``SGLDVideoInput`` + fallback below. """ - video_input = SGLDVideoInput(video_path, height, width) - return video_input + try: + from comfy_api.input_impl import VideoFromFile + + return VideoFromFile(video_path) + except ImportError: + return SGLDVideoInput(video_path, height, width) diff --git a/python/sglang/multimodal_gen/test/unit/test_comfyui_lora_node.py b/python/sglang/multimodal_gen/test/unit/test_comfyui_lora_node.py index c0b6b95a0e..e6145a2d30 100644 --- a/python/sglang/multimodal_gen/test/unit/test_comfyui_lora_node.py +++ b/python/sglang/multimodal_gen/test/unit/test_comfyui_lora_node.py @@ -33,7 +33,13 @@ def _load_nodes(): comfy_api_input = types.ModuleType("comfy_api.input") comfy_api_input.VideoInput = type("VideoInput", (), {}) comfy_api.input = comfy_api_input + comfy = types.ModuleType("comfy") + comfy.model_management = types.ModuleType("comfy.model_management") + comfy.utils = types.ModuleType("comfy.utils") stubs = { + "comfy": comfy, + "comfy.model_management": comfy.model_management, + "comfy.utils": comfy.utils, "folder_paths": folder_paths, "comfy_api": comfy_api, "comfy_api.input": comfy_api_input,