diff --git a/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-add-model/SKILL.md b/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-add-model/SKILL.md index 4f0bb45c1..85450d4af 100644 --- a/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-add-model/SKILL.md +++ b/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-add-model/SKILL.md @@ -574,6 +574,39 @@ After implementation, **you must verify that the generated output is not noise** 2. Running the Diffusers pipeline and SGLang pipeline side-by-side with the same seed 3. Checking each stage's output shape and value range independently +### Step 10: Decide the ComfyUI Route (Optional) + +A model is reachable from ComfyUI two ways. Pick one deliberately — the wrong +choice costs several hundred lines of weight-mapping code that buys nothing. + +**Server route.** ComfyUI sends an HTTP request and SGLang runs the whole +pipeline. Choose this when the model needs conditioning ComfyUI cannot supply +(audio, reference materials, task routing), produces more than one modality, +or has its own request contract. + +Cost: nothing, if the request fits the existing `generate_image` / +`generate_video` fields. If the model has extra request fields, pass them +through `extra_fields` — the request schemas accept unknown keys, so the +client in `apps/ComfyUI_SGLDiffusion/core/server_api.py` does **not** need a +per-model change. Add a node in `nodes.py` only when the inputs are worth +surfacing as ComfyUI widgets. `SGLDiffusionGenerateH3` is the worked example. + +**Executor route.** ComfyUI's KSampler drives the denoise loop and SGLang +replaces the DiT forward, using ComfyUI's own text encoders and VAE. Choose +this only when the model denoises a single latent tensor that ComfyUI already +knows how to build and decode. + +Cost, per model: a `runtime/pipelines/comfyui__pipeline.py` that maps +ComfyUI's single-file checkpoint layout onto the native module tree (350-690 +lines in the existing three), an executor in +`apps/ComfyUI_SGLDiffusion/executors/` that adapts latent layout and +conditioning to `Req`, and entries in both dicts in `core/generator.py`. + +The deciding question is not model size or modality — it is whether ComfyUI's +sampler can drive the model's loop unchanged. If reproducing the conditioning +inside ComfyUI would duplicate stages the server already runs, take the server +route. + ## Reference Implementations ### Hybrid Style (recommended for most new models) diff --git a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/README.md b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/README.md index 305190b68..1fb059efe 100644 --- a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/README.md +++ b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/README.md @@ -16,6 +16,7 @@ The plugin supports two modes of operation: **Server Mode** (via HTTP API) and * - **Z-Image**: High-speed image generation models (e.g., `Z-Image-Turbo`) - **FLUX**: State-of-the-art text-to-image models (e.g., `FLUX.1-dev`) - **Qwen-Image**: Multi-modal image generation models (e.g., `Qwen-Image`,`Qwen-Image-2512`). *Note: Image editing support is currently experimental and may have some issues.* +- **MiniMax-H3**: Joint video-and-audio generation, server mode only (`SGLDiffusion Generate H3`) ### Mode 1: Server Mode (HTTP API) Connect to a standalone SGLang Diffusion server. @@ -35,6 +36,35 @@ Leverage SGLang's high-performance sampling directly within ComfyUI while using 3. **Sample**: Connect the loaded model to standard ComfyUI samplers. SGLang will handle the sampling process efficiently. 4. **LoRA Support**: Use the `SGLDiffusion LoRA Loader` for native LoRA integration. +## Adding a Model + +Pick the mode before writing code; the wrong one costs several hundred lines +of weight mapping that buys nothing. + +Take **Server Mode** when the model needs conditioning ComfyUI cannot supply +(audio, reference materials, task routing), emits more than one modality, or +has its own request contract. Reproducing that inside ComfyUI would duplicate +stages the server already runs. + +- If the request fits the existing `generate_image` / `generate_video` fields, + there is nothing to write — point the existing nodes at the server. +- If the model has extra request fields, pass them via `extra_fields`. The + request schemas accept unknown keys, so `core/server_api.py` needs no + per-model change. +- Add a node in `nodes.py` only to surface those inputs as ComfyUI widgets. + `SGLDiffusionGenerateH3` is the worked example. + +Take **Integrated Mode** only when the model denoises a single latent tensor +that ComfyUI already knows how to build and decode, so its KSampler can drive +the loop unchanged. Each model then needs: + +- `runtime/pipelines/comfyui__pipeline.py` mapping ComfyUI's + single-file checkpoint layout onto the native module tree (350-690 lines in + the existing three) +- an executor in `executors/` adapting latent layout and conditioning to `Req` +- entries in both `pipeline_class_dict` and `executor_class_dict` in + `core/generator.py` + ## Example Workflows Reference workflow files are provided in the `workflows/` directory: diff --git a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/core/server_api.py b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/core/server_api.py index 2f86efe79..0b3f92bec 100644 --- a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/core/server_api.py +++ b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/core/server_api.py @@ -219,6 +219,7 @@ class SGLDiffusionServerAPI: generator_device: Optional[str] = "cuda", input_reference: Optional[str] = None, output_path: Optional[str] = None, + extra_fields: Optional[Dict[str, Any]] = None, ) -> Dict[str, Any]: """ Generate a video using SGLang Diffusion API and wait for completion. @@ -238,6 +239,10 @@ class SGLDiffusionServerAPI: enable_teacache: Enable TEA cache acceleration generator_device: Device for random generator ("cuda" or "cpu") input_reference: Path to input reference image for image-to-video + extra_fields: Model-specific request fields merged into the payload + last, so a caller can reach a model's own request surface + (MiniMax-H3's `task`/`conditions`/`target`, per-model flow + shifts) without this client growing a parameter per model Returns: Dictionary containing completed video job information with file_path @@ -281,6 +286,10 @@ class SGLDiffusionServerAPI: payload["input_reference"] = input_reference if output_path: payload["output_path"] = output_path + # merged last so a model-specific field wins over a generic default of + # the same name (H3 sizes its output from `target`, not `size`) + if extra_fields: + payload.update(extra_fields) try: # Create video generation job diff --git a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/nodes.py b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/nodes.py index af31fe058..f3d2275ee 100644 --- a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/nodes.py +++ b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/nodes.py @@ -576,6 +576,222 @@ class SGLDiffusionGenerateVideo: return (video, video_path) +class SGLDiffusionGenerateH3: + """Node to generate joint video and audio with MiniMax-H3. + + H3 denoises a packed video+audio sequence in one pass and routes its + conditioning by task rather than by a single reference slot, so it needs + its own request shape (`task` / `conditions` / `target`) that the generic + video node does not model. The returned MP4 carries both streams. + """ + + TASKS = ["t2va", "fl2va", "ref2va"] + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "sgld_client": ("SGLD_CLIENT",), + "positive_prompt": ( + "STRING", + { + "default": "", + "tooltip": "Text prompt. Reference material is addressed " + "positionally as ,