From 50ec9702d0dc680d114c311a4df83d96a15062a0 Mon Sep 17 00:00:00 2001 From: Mick Date: Mon, 21 Sep 2026 21:23:11 +0800 Subject: [PATCH] [diffusion] docs: update ComfyUI sections, trimmed examples, and the RTX 5090 DiT-resident recipe (1.42x) for Qwen-Image-2.1 cookbook (#40573) Co-authored-by: Mick Qian Co-authored-by: Claude Opus 5 --- .claude/skills/cookbook-add-model/SKILL.md | 21 ++++ .claude/skills/cookbook-review-pr/SKILL.md | 12 +++ docs/cookbook/diffusion/Cosmos/Cosmos3.mdx | 6 ++ .../diffusion/Ernie-Image/Ernie-Image.mdx | 6 ++ docs/cookbook/diffusion/FLUX/FLUX.mdx | 6 ++ .../cookbook/diffusion/Ideogram/Ideogram4.mdx | 6 ++ docs/cookbook/diffusion/JoyEcho/JoyEcho.mdx | 6 ++ docs/cookbook/diffusion/Krea/Krea-2.mdx | 6 ++ docs/cookbook/diffusion/LTX/LTX2 & LTX2.3.mdx | 6 ++ .../LingBot-World/LingBot-World-2.0.mdx | 6 ++ .../diffusion/LingBot-World/LingBot-World.mdx | 6 ++ .../diffusion/LongLive/LongLive-2.0.mdx | 6 ++ docs/cookbook/diffusion/MOVA/MOVA.mdx | 6 ++ .../cookbook/diffusion/MiniMax/MiniMax-H3.mdx | 6 ++ .../diffusion/Qwen-Image/Qwen-Image-2.1.mdx | 21 ++-- .../diffusion/Qwen-Image/Qwen-Image-Edit.mdx | 6 ++ .../diffusion/Qwen-Image/Qwen-Image.mdx | 6 ++ docs/cookbook/diffusion/SANA-WM/SANA-WM.mdx | 6 ++ docs/cookbook/diffusion/Wan/Wan2.1.mdx | 6 ++ docs/cookbook/diffusion/Wan/Wan2.2.mdx | 6 ++ .../diffusion/Z-Image/Z-Image-Turbo.mdx | 6 ++ .../snippets/configs/Qwen/qwen-image-2.1.jsx | 21 ++-- .../snippets/diffusion/comfyui-support.jsx | 95 +++++++++++++++++++ 23 files changed, 256 insertions(+), 22 deletions(-) create mode 100644 docs/src/snippets/diffusion/comfyui-support.jsx diff --git a/.claude/skills/cookbook-add-model/SKILL.md b/.claude/skills/cookbook-add-model/SKILL.md index 2d5e722fd..803ae11c7 100644 --- a/.claude/skills/cookbook-add-model/SKILL.md +++ b/.claude/skills/cookbook-add-model/SKILL.md @@ -169,6 +169,27 @@ least one real deployment or capability boundary. Put orthogonal runtime feature variant/quant/strategy needs its own image), `multiNodeHints` only for fabric-specific hw (e.g. gb200). +5. **Diffusion pages: add the ComfyUI section.** Every diffusion cookbook page ends with + a `## . Run in ComfyUI` section so a reader never has to guess whether the model is + reachable from ComfyUI. It is one component; the per-model facts live in the component, + not the page: + + ```mdx + ## . Run in ComfyUI + + import { ComfyUISupport } from '/src/snippets/diffusion/comfyui-support.jsx'; + + + ``` + + Pick `model` from the table in + `docs/src/snippets/diffusion/comfyui-support.jsx`. Use the model's own key when it has + an entry (its executor or dedicated node differs); otherwise use the generic `image` or + `video`. A model gets its own key only when the plugin actually treats it specially — + an entry in `executor_class_dict` + (`python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/core/generator.py`) or a + dedicated node. Adding a key without the matching plugin support makes the page lie. + ### Site-wiring (do all three) - **`docs/docs.json`** — add the page under Cookbook → `` → ``, at diff --git a/.claude/skills/cookbook-review-pr/SKILL.md b/.claude/skills/cookbook-review-pr/SKILL.md index d7ccaf46a..27d29f087 100644 --- a/.claude/skills/cookbook-review-pr/SKILL.md +++ b/.claude/skills/cookbook-review-pr/SKILL.md @@ -147,6 +147,18 @@ than restating. equal what the engine emits from the corresponding cell — same flags, same order. Drift here is the most common review miss. +### 5b. ComfyUI section (diffusion pages) +- A diffusion page ends with `## . Run in ComfyUI` rendering ``. A + reader must not have to guess whether the model is reachable from ComfyUI. +- The `model` prop is a key that exists in `docs/src/snippets/diffusion/comfyui-support.jsx`. + A model-specific key is only correct when the plugin really treats it specially — an entry + in `executor_class_dict` + (`python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/core/generator.py`) or a dedicated + node. Otherwise the generic `image` / `video` key is the honest one; a model-specific key + without matching plugin support makes the page claim support that does not exist. +- Prose describing ComfyUI support inline instead of using the component is a finding: the + facts drift from the plugin. + ### 6. Commands / port - Launch uses `sglang serve` — flag any `python -m sglang.launch_server` / `python3 -m sglang.launch_server` (deprecated). The engine already emits `sglang serve`; diff --git a/docs/cookbook/diffusion/Cosmos/Cosmos3.mdx b/docs/cookbook/diffusion/Cosmos/Cosmos3.mdx index 3239274e2..7efeeed4d 100644 --- a/docs/cookbook/diffusion/Cosmos/Cosmos3.mdx +++ b/docs/cookbook/diffusion/Cosmos/Cosmos3.mdx @@ -400,3 +400,9 @@ remains accepted for compatibility, but new clients should use `extra_body`: - `use_resolution_template`: accepted for vLLM-Omni request compatibility. - `use_system_prompt`: whether to add the Cosmos3 system prompt to the chat template. - `guardrails` or `use_guardrails`: per-request guardrail toggle when the server started with guardrails enabled. + +## 6. Run in ComfyUI + +import { ComfyUISupport } from '/src/snippets/diffusion/comfyui-support.jsx'; + + diff --git a/docs/cookbook/diffusion/Ernie-Image/Ernie-Image.mdx b/docs/cookbook/diffusion/Ernie-Image/Ernie-Image.mdx index a4c420a07..e8c666809 100644 --- a/docs/cookbook/diffusion/Ernie-Image/Ernie-Image.mdx +++ b/docs/cookbook/diffusion/Ernie-Image/Ernie-Image.mdx @@ -81,3 +81,9 @@ with open("ernie_image.png", "wb") as f: - `--performance-mode auto` keeps conservative defaults while preserving explicit user flags. - If the checkpoint includes a PE component, SGLang loads it automatically with the native Ministral3 runtime. Use `--layerwise-offload-components pe` when the local PE decoder needs to trade latency for lower GPU memory usage. - Treat FSDP, SP/Ulysses/Ring, and TP as explicit benchmark knobs. Measure the target resolution, step count, and GPU type before making them production defaults. + +## 6. Run in ComfyUI + +import { ComfyUISupport } from '/src/snippets/diffusion/comfyui-support.jsx'; + + diff --git a/docs/cookbook/diffusion/FLUX/FLUX.mdx b/docs/cookbook/diffusion/FLUX/FLUX.mdx index 78a659abd..4cb84c7d8 100644 --- a/docs/cookbook/diffusion/FLUX/FLUX.mdx +++ b/docs/cookbook/diffusion/FLUX/FLUX.mdx @@ -407,3 +407,9 @@ Test Environment: ``` + +## 6. Run in ComfyUI + +import { ComfyUISupport } from '/src/snippets/diffusion/comfyui-support.jsx'; + + diff --git a/docs/cookbook/diffusion/Ideogram/Ideogram4.mdx b/docs/cookbook/diffusion/Ideogram/Ideogram4.mdx index c02530e6a..9e17b3b27 100644 --- a/docs/cookbook/diffusion/Ideogram/Ideogram4.mdx +++ b/docs/cookbook/diffusion/Ideogram/Ideogram4.mdx @@ -187,3 +187,9 @@ response = client.images.generate( ``` Base Ideogram 4 presets are `V4_DEFAULT_20`, `V4_QUALITY_48`, and `V4_TURBO_12`. The fal variants automatically select `V4_FAST_20` and `V4_INSTANT_8`, respectively. A preset controls both `num_inference_steps` and guidance, so do not set those fields directly. + +## 5. Run in ComfyUI + +import { ComfyUISupport } from '/src/snippets/diffusion/comfyui-support.jsx'; + + diff --git a/docs/cookbook/diffusion/JoyEcho/JoyEcho.mdx b/docs/cookbook/diffusion/JoyEcho/JoyEcho.mdx index 2c17476fa..f41d7b881 100644 --- a/docs/cookbook/diffusion/JoyEcho/JoyEcho.mdx +++ b/docs/cookbook/diffusion/JoyEcho/JoyEcho.mdx @@ -238,3 +238,9 @@ Use `quality=lossless` for this recipe. High-mode output did not pass the separa - For **2-GPU latency**, try **Ulysses SP** (`--num-gpus 2 --ulysses-degree 2`) on both single-shot and multi-shot runs. Use **TP** when you need a different sharding strategy or more than two GPUs. - Set `PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True` for long multi-shot SP sessions. - JoyEcho outputs per-shot mp4 files with synchronized audio. There is no built-in two-stage HQ upscaling path like LTX-2.3 HQ. + +## 6. Run in ComfyUI + +import { ComfyUISupport } from '/src/snippets/diffusion/comfyui-support.jsx'; + + diff --git a/docs/cookbook/diffusion/Krea/Krea-2.mdx b/docs/cookbook/diffusion/Krea/Krea-2.mdx index 1048c80ec..afc40cfea 100644 --- a/docs/cookbook/diffusion/Krea/Krea-2.mdx +++ b/docs/cookbook/diffusion/Krea/Krea-2.mdx @@ -334,3 +334,9 @@ Peak Memory Mean (MB): 37466.40 Peak Memory Median (MB): 37466.00 ============================================================ ``` + +## 6. Run in ComfyUI + +import { ComfyUISupport } from '/src/snippets/diffusion/comfyui-support.jsx'; + + diff --git a/docs/cookbook/diffusion/LTX/LTX2 & LTX2.3.mdx b/docs/cookbook/diffusion/LTX/LTX2 & LTX2.3.mdx index 56f22bafe..99f4f11c8 100644 --- a/docs/cookbook/diffusion/LTX/LTX2 & LTX2.3.mdx +++ b/docs/cookbook/diffusion/LTX/LTX2 & LTX2.3.mdx @@ -248,3 +248,9 @@ Some community LoRAs only include weights for transformer blocks. In that case, - Use `--ltx2-two-stage-device-mode resident` on high-VRAM GPUs if latency matters more than memory usage. - Use `--ltx2-two-stage-device-mode original` when comparing against official two-stage behavior. - Keep `--width` and `--height` aligned with the target model resolution; for LTX models, these are output video dimensions. + +## 6. Run in ComfyUI + +import { ComfyUISupport } from '/src/snippets/diffusion/comfyui-support.jsx'; + + diff --git a/docs/cookbook/diffusion/LingBot-World/LingBot-World-2.0.mdx b/docs/cookbook/diffusion/LingBot-World/LingBot-World-2.0.mdx index fae0cc941..384919e1a 100644 --- a/docs/cookbook/diffusion/LingBot-World/LingBot-World-2.0.mdx +++ b/docs/cookbook/diffusion/LingBot-World/LingBot-World-2.0.mdx @@ -184,3 +184,9 @@ LingBot World 2.0 uses raw-frame websocket GT plus per-chunk latency guards for - Use the realtime endpoint for interactive sessions: `/v1/realtime_video/generate`. - Prefer WebP preview transport for interactive testing; use raw-frame transport for consistency checks. - Long-running sessions should be validated with raw-frame consistency before changing causal cache, condition sampling, or VAE decode behavior. + +## 7. Run in ComfyUI + +import { ComfyUISupport } from '/src/snippets/diffusion/comfyui-support.jsx'; + + diff --git a/docs/cookbook/diffusion/LingBot-World/LingBot-World.mdx b/docs/cookbook/diffusion/LingBot-World/LingBot-World.mdx index b6927bc38..a3dea5325 100644 --- a/docs/cookbook/diffusion/LingBot-World/LingBot-World.mdx +++ b/docs/cookbook/diffusion/LingBot-World/LingBot-World.mdx @@ -159,3 +159,9 @@ LingBot World uses raw-frame websocket GT plus per-chunk latency guards for cons - Use the realtime endpoint for interactive sessions: `/v1/realtime_video/generate`. - Prefer WebP preview transport for interactive testing; use raw-frame transport for consistency checks. - Long-running sessions should be validated with raw-frame consistency before changing causal cache, condition sampling, or VAE decode behavior. + +## 7. Run in ComfyUI + +import { ComfyUISupport } from '/src/snippets/diffusion/comfyui-support.jsx'; + + diff --git a/docs/cookbook/diffusion/LongLive/LongLive-2.0.mdx b/docs/cookbook/diffusion/LongLive/LongLive-2.0.mdx index 4da75d45c..0d9a8363d 100644 --- a/docs/cookbook/diffusion/LongLive/LongLive-2.0.mdx +++ b/docs/cookbook/diffusion/LongLive/LongLive-2.0.mdx @@ -117,3 +117,9 @@ The image is used as the first-frame condition. - SGLang supports T2V sizes 1280x704, 704x1280, 832x480, and 480x832. - I2V request images follow the Wan TI2V preprocessing path in SGLang. This is different from the original LongLive dataset resize path. - For multi-shot runs, set `num_frames` to match `len(shot_prompts) * chunks_per_shot * 8` latent frames, that is `num_frames = (len(shot_prompts) * chunks_per_shot * 8 - 1) * 4 + 1`. + +## 6. Run in ComfyUI + +import { ComfyUISupport } from '/src/snippets/diffusion/comfyui-support.jsx'; + + diff --git a/docs/cookbook/diffusion/MOVA/MOVA.mdx b/docs/cookbook/diffusion/MOVA/MOVA.mdx index d6d7779ee..2f9c9aa74 100644 --- a/docs/cookbook/diffusion/MOVA/MOVA.mdx +++ b/docs/cookbook/diffusion/MOVA/MOVA.mdx @@ -266,3 +266,9 @@ python3 -m sglang.multimodal_gen.benchmarks.bench_serving \ --task image-to-video --dataset vbench --num-prompts 20 --max-concurrency 20 \ --port 30002 ``` + +## 6. Run in ComfyUI + +import { ComfyUISupport } from '/src/snippets/diffusion/comfyui-support.jsx'; + + diff --git a/docs/cookbook/diffusion/MiniMax/MiniMax-H3.mdx b/docs/cookbook/diffusion/MiniMax/MiniMax-H3.mdx index 38ef972e0..296cc7e81 100644 --- a/docs/cookbook/diffusion/MiniMax/MiniMax-H3.mdx +++ b/docs/cookbook/diffusion/MiniMax/MiniMax-H3.mdx @@ -2055,3 +2055,9 @@ For a measured lower-count AMD deployment, set both `--num-gpus` and `--ulysses-degree` to 4, 2, or 1. AITER packed attention matched segment-wise BF16 SDPA at cosine similarity `0.9999991655` on MI355X and `0.9999991059` on MI300X. + +## 10. Run in ComfyUI + +import { ComfyUISupport } from '/src/snippets/diffusion/comfyui-support.jsx'; + + diff --git a/docs/cookbook/diffusion/Qwen-Image/Qwen-Image-2.1.mdx b/docs/cookbook/diffusion/Qwen-Image/Qwen-Image-2.1.mdx index d5c705171..46cc9ad90 100644 --- a/docs/cookbook/diffusion/Qwen-Image/Qwen-Image-2.1.mdx +++ b/docs/cookbook/diffusion/Qwen-Image/Qwen-Image-2.1.mdx @@ -52,6 +52,8 @@ PY The picker defaults to native BF16/FP32 precision, exact attention, eager execution, and full-image VAE decoding. +Commands omit default values, including one GPU, encoder auto scheduling, and +batch size one. Explicit placement and attention overrides preserve each recipe. | GPU | Placement / attention | Generation | Edit | Peak VRAM | | --- | --- | --- | --- | --- | @@ -140,29 +142,24 @@ including partly transparent edges, without thresholding or background removal. ## 4. Offline requests +Defaults are 1024×1024, 40 steps, CFG 1, and seed 42; output saving is enabled. +For GPUs that need offload, also pass the placement flags from the picker. + ### Text-to-image ```bash Command sglang generate \ - --model-path /models/qwen-image-2.1 \ - --model-id Qwen-Image-2.1 \ - --prompt "A capybara reading a book by candlelight" \ - --width 1024 --height 1024 \ - --num-inference-steps 40 --guidance-scale 1 \ - --seed 0 --save-output + --model-path Qwen/Qwen-Image-2.1 \ + --prompt "A capybara reading a book by candlelight" ``` ### Image-conditioned editing ```bash Command sglang generate \ - --model-path /models/qwen-image-2.1 \ - --model-id Qwen-Image-2.1 \ + --model-path Qwen/Qwen-Image-2.1 \ --image-path /path/to/input.png \ - --prompt "Move the scene to a snowy mountain at sunrise" \ - --width 1024 --height 1024 \ - --num-inference-steps 40 --guidance-scale 1 \ - --seed 0 --save-output + --prompt "Move the scene to a snowy mountain at sunrise" ``` Height and width must be positive multiples of 32. Reference images preserve diff --git a/docs/cookbook/diffusion/Qwen-Image/Qwen-Image-Edit.mdx b/docs/cookbook/diffusion/Qwen-Image/Qwen-Image-Edit.mdx index 5479fa733..5b7c92e1e 100644 --- a/docs/cookbook/diffusion/Qwen-Image/Qwen-Image-Edit.mdx +++ b/docs/cookbook/diffusion/Qwen-Image/Qwen-Image-Edit.mdx @@ -315,3 +315,9 @@ Peak Memory Mean (MB): 47971.49 Peak Memory Median (MB): 47971.29 ============================================================ ``` + +## 6. Run in ComfyUI + +import { ComfyUISupport } from '/src/snippets/diffusion/comfyui-support.jsx'; + + diff --git a/docs/cookbook/diffusion/Qwen-Image/Qwen-Image.mdx b/docs/cookbook/diffusion/Qwen-Image/Qwen-Image.mdx index 87d6e1320..881dfa9c5 100644 --- a/docs/cookbook/diffusion/Qwen-Image/Qwen-Image.mdx +++ b/docs/cookbook/diffusion/Qwen-Image/Qwen-Image.mdx @@ -428,3 +428,9 @@ Test Environment: ``` + +## 6. Run in ComfyUI + +import { ComfyUISupport } from '/src/snippets/diffusion/comfyui-support.jsx'; + + diff --git a/docs/cookbook/diffusion/SANA-WM/SANA-WM.mdx b/docs/cookbook/diffusion/SANA-WM/SANA-WM.mdx index 811e13b6a..4983ea663 100644 --- a/docs/cookbook/diffusion/SANA-WM/SANA-WM.mdx +++ b/docs/cookbook/diffusion/SANA-WM/SANA-WM.mdx @@ -487,3 +487,9 @@ At WebSocket `init` the realtime adapter fills SANA-WM defaults that differ from `guidance_scale` applies to the dense path (§4) only; the distilled streaming path uses `streaming_cfg_scale` (default `1.0`, i.e. no CFG) so a `guidance_scale` override never accidentally enables CFG on the streaming stage. `denoising_step_list = (1000, 960, 889, 727, 0)` is the official 4-step streaming schedule (it must end in 0). + +## 10. Run in ComfyUI + +import { ComfyUISupport } from '/src/snippets/diffusion/comfyui-support.jsx'; + + diff --git a/docs/cookbook/diffusion/Wan/Wan2.1.mdx b/docs/cookbook/diffusion/Wan/Wan2.1.mdx index a517902c9..2c5640526 100644 --- a/docs/cookbook/diffusion/Wan/Wan2.1.mdx +++ b/docs/cookbook/diffusion/Wan/Wan2.1.mdx @@ -381,3 +381,9 @@ You can use the built-in SGLang diffusion benchmark script to evaluate Wan2.1 pe ``` + +## 6. Run in ComfyUI + +import { ComfyUISupport } from '/src/snippets/diffusion/comfyui-support.jsx'; + + diff --git a/docs/cookbook/diffusion/Wan/Wan2.2.mdx b/docs/cookbook/diffusion/Wan/Wan2.2.mdx index d2b8f6c9f..a24355ed3 100644 --- a/docs/cookbook/diffusion/Wan/Wan2.2.mdx +++ b/docs/cookbook/diffusion/Wan/Wan2.2.mdx @@ -459,3 +459,9 @@ Test Environment: ``` + +## 6. Run in ComfyUI + +import { ComfyUISupport } from '/src/snippets/diffusion/comfyui-support.jsx'; + + diff --git a/docs/cookbook/diffusion/Z-Image/Z-Image-Turbo.mdx b/docs/cookbook/diffusion/Z-Image/Z-Image-Turbo.mdx index 69b3b662c..b5dc01e4a 100644 --- a/docs/cookbook/diffusion/Z-Image/Z-Image-Turbo.mdx +++ b/docs/cookbook/diffusion/Z-Image/Z-Image-Turbo.mdx @@ -366,3 +366,9 @@ Test Environment: ``` + +## 6. Run in ComfyUI + +import { ComfyUISupport } from '/src/snippets/diffusion/comfyui-support.jsx'; + + diff --git a/docs/src/snippets/configs/Qwen/qwen-image-2.1.jsx b/docs/src/snippets/configs/Qwen/qwen-image-2.1.jsx index 211bc45f8..460fcd017 100644 --- a/docs/src/snippets/configs/Qwen/qwen-image-2.1.jsx +++ b/docs/src/snippets/configs/Qwen/qwen-image-2.1.jsx @@ -53,14 +53,14 @@ const config = { }, { id: "offload", label: "CPU offload", - flags: (s) => s.hw === "rtx4090" && Number(s.gpus_per_node) === 1 && effectiveAttention(s) === "fa" && s.precision === "native" && s.execution === "eager" + flags: (s) => ["rtx4090", "rtx5090"].includes(s.hw) && Number(s.gpus_per_node) === 1 && effectiveAttention(s) === platformAttention(s) && s.precision === "native" && s.execution === "eager" && ["text", "edit"].includes(s.mode) && Number(s.outputs) === 1 && (!s.batching || s.batching === "off") - ? ["--performance-mode manual", "--component-residency dit=resident text_encoder=layerwise-offload vae=resident", `--warmup-resolutions ${s.resolution || "1024"}x${s.resolution || "1024"}`] + ? ["--performance-mode manual", "--component-residency text_encoder=layerwise-offload"] : ["--performance-mode manual", "--dit-layerwise-offload true", ...(s.hw === "rtx4090" ? ["--text-encoder-cpu-offload true"] : [])], recommendedWhen: (s) => ["rtx5090", "rtx4090"].includes(s.hw), soft: (s) => !["rtxpro6000", "rtx5090", "rtx4090"].includes(s.hw) || Number(s.gpus_per_node) !== 1, softReason: "This offload topology has not completed an HTTP verification run.", - description: "RTX 4090 native single-output FlashAttention keeps the DiT and VAE resident and streams encoder layers. Other offload recipes stream DiT layers; RTX 4090 also offloads the encoder. Requires sufficient host RAM.", + description: "RTX 4090 and RTX 5090 keep the DiT and VAE on the card, on their platform attention kernel, and stream encoder layers. That is faster than streaming the DiT: measured 1024px / 40 steps on one RTX 5090, 14.12s against 19.95s, on 17.0GB against 19.3GB steady. Other offload recipes stream DiT layers. Requires sufficient host RAM.", }, { id: "all_offload", label: "All components layerwise", @@ -210,8 +210,8 @@ const config = { { id: "eager", label: "Eager", recommended: true }, { id: "bcg", label: "Breakable CUDA Graph", - flags: (s) => ["--enable-breakable-cuda-graph true", `--warmup-resolutions ${s.resolution || "1024"}x${s.resolution || "1024"}`, "--bcg-text-buckets 64"], - soft: true, softReason: "A 1024px H200 server captured its warmup graph, but tested requests fell back to eager because condition-prefix shapes differed.", + flags: (s) => ["--enable-breakable-cuda-graph true", `--warmup-resolutions ${s.resolution || "1024"}x${s.resolution || "1024"}`], + soft: true, softReason: "Unmatched condition-prefix shapes run eagerly. Keep eager execution for the recommended recipes.", description: "Captures the selected resolution. Condition-prefix shapes must also match warmup; text buckets alone do not ensure replay.", }, ], @@ -398,16 +398,17 @@ const config = { : "Combine the subjects from Picture 1 and Picture 2 into one coherent scene, preserving their appearance.", }; const request = { - model: "{{MODEL_NAME}}", prompt: prompts[s.mode], n: Number(s.outputs), - size: `${s.resolution}x${s.resolution}`, num_inference_steps: Number(s.steps), - guidance_scale: 1, seed: 42, generator_device: "cpu", + prompt: prompts[s.mode], generator_device: "cpu", output_format: "png", response_format: "b64_json", - background: transparent ? "transparent" : "auto", }; + if (Number(s.outputs) !== 1) request.n = Number(s.outputs); + if (s.resolution !== "1024") request.size = `${s.resolution}x${s.resolution}`; + if (Number(s.steps) !== 40) request.num_inference_steps = Number(s.steps); + if (transparent) request.background = "transparent"; if (s.mode === "text") { return `curl -sS --fail-with-body http://{{CURL_HOST}}:{{CURL_PORT}}/v1/images/generations \\ -H 'Content-Type: application/json' \\ - -d '${JSON.stringify({ ...request, enable_cache_dit: false }, null, 2)}'`; + -d '${JSON.stringify(request, null, 2)}'`; } const fields = Object.entries(request).map(([key, value]) => ` --form-string '${key}=${value}'`); fields.push(' -F "image[]=@{{INPUT_IMAGE}};type=image/png"'); diff --git a/docs/src/snippets/diffusion/comfyui-support.jsx b/docs/src/snippets/diffusion/comfyui-support.jsx new file mode 100644 index 000000000..a2540f4c8 --- /dev/null +++ b/docs/src/snippets/diffusion/comfyui-support.jsx @@ -0,0 +1,95 @@ +// ComfyUI support statement for a diffusion cookbook page. +// +// The per-model facts live here rather than in each page so they stay in sync +// with the plugin. `integrated` must match a key in `executor_class_dict` +// (python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/core/generator.py); +// a model absent from that dict has no integrated mode. + +const PLUGIN_PATH = "python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion"; +const PLUGIN_URL = + "https://github.com/sgl-project/sglang/tree/main/" + PLUGIN_PATH; + +const MODELS = { + flux: { + serverNode: "SGLDiffusion Generate Image", + integratedKey: "flux", + workflow: "flux_sgld_sp.json", + }, + "z-image": { + serverNode: "SGLDiffusion Generate Image", + integratedKey: "lumina2", + workflow: "z-image_sgld.json", + }, + "qwen-image": { + serverNode: "SGLDiffusion Generate Image", + integratedKey: "qwen_image", + workflow: "qwen_image_sgld.json", + }, + "qwen-image-edit": { + serverNode: "SGLDiffusion Generate Image", + integratedKey: "qwen_image_edit", + note: "Image editing through the integrated path is experimental.", + }, + "minimax-h3": { + serverNode: "SGLDiffusion Generate H3", + verified: true, + integratedBlockedBecause: + "H3 denoises a packed video-and-audio sequence in one pass and routes " + + "conditioning by task, while ComfyUI's KSampler drives a single latent " + + "tensor and has no audio branch", + }, + image: { serverNode: "SGLDiffusion Generate Image" }, + video: { serverNode: "SGLDiffusion Generate Video" }, +}; + +export const ComfyUISupport = ({ model = "video", note }) => { + const spec = MODELS[model] || MODELS.video; + const extraNote = note || spec.note; + + return ( +
+

+ Run this model from ComfyUI with the{" "} + SGLDiffusion plugin, which ships in the SGLang + repository at {PLUGIN_PATH}. +

+ +

+ Server mode — SGLang runs the pipeline and ComfyUI + sends the request. Start a server as shown above, point the{" "} + SGLDiffusion Server Model node at it, then generate with{" "} + {spec.serverNode}. + {spec.verified + ? " This path has been run end to end against a live server." + : ""} +

+ + {spec.integratedKey ? ( +

+ Integrated mode — ComfyUI's own sampler, CLIP, and + VAE drive the loop while SGLang replaces the model forward. Load the + checkpoint with SGLDiffusion UNET Loader and set{" "} + model_type to {spec.integratedKey} on the{" "} + SGLDiffusion Options node. + {spec.workflow ? ( + <> + {" "} + A reference workflow is included at{" "} + {`${PLUGIN_PATH}/workflows/${spec.workflow}`}. + + ) : null} +

+ ) : ( +

+ Integrated mode — not available for this model + {spec.integratedBlockedBecause + ? `: ${spec.integratedBlockedBecause}` + : ", which has no executor in the plugin"} + . Use server mode. +

+ )} + + {extraNote ?

{extraNote}

: null} +
+ ); +};