From b43931e8784e724a0afd8222fcab81ec803a2bba Mon Sep 17 00:00:00 2001 From: Xiaoyu Zhang <1182563586@qq.com> Date: Mon, 24 Aug 2026 13:37:45 +0800 Subject: [PATCH] [diffusion] Refresh quality and BCG benchmark skills (#36016) Signed-off-by: BBuf <1182563586@qq.com> --- .../sglang-diffusion-add-model/SKILL.md | 71 ++ .../SKILL.md | 42 +- .../benchmark-and-profile.md | 118 ++- .../existing-fast-paths.md | 40 +- .../scripts/bench_diffusion_denoise.py | 789 +++++++++++++++++- .../scripts/diffusion_skill_env.py | 8 + .../sglang-diffusion-performance/SKILL.md | 40 +- .../unit/test_diffusion_benchmark_skill.py | 306 +++++++ 8 files changed, 1370 insertions(+), 44 deletions(-) diff --git a/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-add-model/SKILL.md b/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-add-model/SKILL.md index 85450d4af..2d07edac5 100644 --- a/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-add-model/SKILL.md +++ b/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-add-model/SKILL.md @@ -607,6 +607,69 @@ sampler can drive the model's loop unchanged. If reproducing the conditioning inside ComfyUI would duplicate stages the server already runs, take the server route. +### Step 11: Opt In to BCG and Quality Fast Paths Only After Eager Parity + +Do not put a new model behind Breakable CUDA Graph (BCG) merely because one +forward captures. Diffusion BCG support has three independent admission paths: + +1. register the exact model IDs and safe basename aliases in + `BREAKABLE_CUDA_GRAPH_SUPPORTED_MODEL_IDS` +2. register the resolved pipeline config class in + `BREAKABLE_CUDA_GRAPH_SUPPORTED_PIPELINE_CONFIGS` +3. implement or select the correct prompt padder under + `runtime/breakable_cuda_graph/model_padders/` + +The third item is model semantics, not a generic shape utility. Reuse +`pad_masked_prompt_kwargs` only when the model already consumes a real mask and +zero-padding every coupled text tensor leaves attention and RoPE unchanged. +Existing special cases show the common contracts: + +- Qwen pads embeddings, masks, text RoPE caches, and sequence-length metadata + together; it synthesizes a mask when the eager path did not need one. +- Ideogram pads the combined text-image sequence and carries replay-local + `DynamicVarlenMaskMeta`; stale capture-time varlen indices are incorrect. +- Z-Image preserves native prompt length because extra tokens change its + semantics even when the padding looks conventional. +- MiniMax-H3 buckets only within compatible packed-sequence alignment groups. +- LongCat-Image and default SANA-Video already produce fixed 512- and + 300-token contracts, respectively, so their padders are pass-through. + +Keep mask construction active for batch size one. A shortcut such as +`if batch > 1` can make eager B=1 appear valid while BCG B=1 attends padded +tokens. Any object whose values depend on live lengths must be rebuilt from +static replay buffers once per replay; do not bake Python lists, varlen +indices, or weakly referenced tensors from warmup into the graph. + +BCG validation must prove all of the following: + +- warmup logs `[Diffusion BCG] captured` +- serving logs no support disable, capture failure, or + `serving signature MISSED` +- lossless Eager and BCG artifacts are byte-identical for the same prompt, + seed, shape, steps, guidance, dtype, and topology +- short/long prompts exercise every intended bucket and an over-limit prompt + falls back deliberately +- video frame count and conditioning shapes match the captured signature; + `--warmup-resolutions` specifies only width and height +- padder and support-gate unit tests cover aliases, pipeline config, fixed + lengths, masks, RoPE/position tensors, and replay-local metadata + +For a non-bit-exact optimization, integrate through the request-scoped site +framework under `sglang.kernels.ops.diffusion.sites`. Mark sites during model +construction and let `QualityGatedFusion` mount them only for +`quality="high"`; `quality="lossless"` must keep the original code path. +Eligibility must be all-or-nothing for coupled sites and fail closed on dtype, +shape, layout, backend, BCG, or compile incompatibility. Add clean site-level +guard/parity tests and a model wiring test instead of embedding request-policy +branches throughout the DiT. + +Finally, use the benchmark/profile skill's `--quality-bcg-matrix` to run +same-GPU ABBA pairs for Eager/BCG at lossless/high. Report denoise and saved +request e2e separately, require at least 1.5% repeated mean e2e improvement for +an optimization PR, attach profile and generated-media A/B evidence, then +delete the task-owned checkpoint cache and verify zero residual weight files +in the cleanup ledger. + ## Reference Implementations ### Hybrid Style (recommended for most new models) @@ -671,6 +734,14 @@ Before submitting, verify: - [ ] Weight names match Diffusers for automatic loading - [ ] **TP/SP support** considered for DiT model (recommended; reference `wanvideo.py` for TP+SP, `qwen_image.py` for USPAttention) - [ ] **Output quality verified** — generated images/videos are not noise; compared against Diffusers reference output +- [ ] **BCG admission is complete or intentionally absent** — model ID, + pipeline config, and model-specific padding contract agree +- [ ] **BCG replay is proven when enabled** — capture marker present, no + signature miss/fallback, and lossless artifact hash is exact +- [ ] **Quality fast paths are request-scoped** — lossless remains untouched; + high-quality sites fail closed and have guard/parity tests +- [ ] **Performance evidence is controlled** — same-GPU repeated e2e, profile, + generated-media comparison, and task-owned weight cleanup ledger **Hybrid style only:** - [ ] **BeforeDenoisingStage** at `stages/model_specific_stages/{model_name}.py` diff --git a/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-benchmark-profile/SKILL.md b/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-benchmark-profile/SKILL.md index d35121d2c..eafd9ed7d 100644 --- a/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-benchmark-profile/SKILL.md +++ b/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-benchmark-profile/SKILL.md @@ -9,6 +9,7 @@ Use this skill when measuring denoise performance, finding the slow op, checking This skill is diagnosis-first. It owns: - checked-in denoise benchmark presets +- same-GPU quality/BCG applicability checks with repeated lossless and high rows - perf dump collection and before/after comparison - `torch.profiler` trace capture and quick hotspot ranking - mapping hot kernels back to known fast paths and fusion families @@ -29,7 +30,8 @@ Before running any benchmark, profiler, or kernel-validation command: - for downloaded checkpoints, use the preset helper's task-owned `--model-cache-root` together with `--cleanup-model-cache`; verify the JSONL ledger reports zero residual weight files before moving to the next model -- choose idle GPU(s) before starting perf work +- choose idle GPU(s) before starting perf work; for a comparison matrix, hold + the same GPU set and verify it has no foreign process at every run boundary ## Native Backend Gate @@ -48,10 +50,10 @@ If any benchmark, perf-dump, or `torch.profiler` command prints one of those sig ## Main Reference -- [benchmark-and-profile.md](benchmark-and-profile.md) — canonical denoise benchmark, perf dump, and `torch.profiler` workflow; uses checked-in nightly-aligned presets plus current-source extras such as LongCat-Image, SANA-Video, LingBot Video MoE, Cosmos3 Edge/distilled, LTX-2.5 and its diffusion decoder, MiniMax-H3, FLUX.2 Klein, Ideogram4, ERNIE/GLM/SANA image models, FastWan2.2, `LTX-2.3`, HunyuanVideo, MOVA, Helios, image edit, and Hunyuan3D shape +- [benchmark-and-profile.md](benchmark-and-profile.md) — canonical denoise benchmark, perf dump, and `torch.profiler` workflow; uses checked-in nightly-aligned presets plus current-source extras such as LongCat-Image, SANA-Video/SANA-WM, LingBot Video/World, Cosmos3 Edge/Super I2V/distilled and the explicit Super TP2 x CFG2 comparator, LTX-2.5 and its diffusion decoder, MiniMax-H3, FLUX.2 Klein, Ideogram4, ERNIE/GLM/SANA image models, FastWan2.1/2.2, the Blackwell-only Wan2.2 NVFP4 comparator, `LTX-2.3`, HunyuanVideo, MOVA, Helios, image edit, and Hunyuan3D shape - [existing-fast-paths.md](existing-fast-paths.md) — map bottlenecks to existing fused kernels, packed QKV paths, fused `QK norm + RoPE`, distributed overlap patterns, and open optimization PRs before proposing new code -- [scripts/diffusion_skill_env.py](scripts/diffusion_skill_env.py) — preflight helper: repo root discovery via `sglang.__file__`, write-access probe, benchmark/profile output directories, idle GPU selection -- [scripts/bench_diffusion_denoise.py](scripts/bench_diffusion_denoise.py) — end-to-end denoise benchmark preset runner via `sglang generate`; defaults to eager, supports opt-in `--torch-compile`, forces H3 to its eager consistency mode, enables synchronized stage attribution, validates nightly preset drift, and can clean an isolated model cache in a `finally` block with a JSONL ledger +- [scripts/diffusion_skill_env.py](scripts/diffusion_skill_env.py) — preflight helper: repo root discovery from the skill's owning checkout before falling back to `sglang.__file__`, write-access probe, benchmark/profile output directories, idle GPU selection +- [scripts/bench_diffusion_denoise.py](scripts/bench_diffusion_denoise.py) — end-to-end denoise benchmark preset runner via `sglang generate`; defaults to eager/lossless, supports explicit quality and BCG comparators plus a same-GPU applicability matrix, rejects invalid BCG capture/fallback logs and late high-quality DiT fusion mounts, forces H3 to its eager consistency mode, enables synchronized stage attribution, validates nightly preset drift, and can clean one isolated model cache after the full matrix in a `finally` block with a JSONL ledger ## Opportunity Discovery Rule @@ -91,9 +93,39 @@ The checked-in helper defaults to eager. Use `--torch-compile` only for a controlled comparator, never for the eager ground truth. The legacy `--no-torch-compile` spelling remains accepted but is redundant. +For kernel/BCG discovery, run `--quality-bcg-matrix`. It executes Eager/BCG as +A-B-B-A at `lossless`, then repeats the pair at `high`, on one locked GPU set +and one isolated checkpoint cache. The high+BCG rows are applicability checks, +not presumed-valid performance cells. A BCG row is invalid unless the log +contains `[Diffusion BCG] captured` and contains no support-disable, +capture-failure, serving-signature-miss, or late quality-fusion marker. In +particular, a request-scoped DiT fusion mounted after lossless warmup capture +would be bypassed by replay; reject that row even when capture and signature +checks pass. `--warmup-resolutions` only declares width and height: a video +request can still miss because its frame count differs from the model's +synthetic warmup contract. Treat that as Eager fallback, not as a valid BCG +measurement. + +A zero process exit is not sufficient evidence: every accepted row must also +contain its requested perf dump and a generated image, video, or audio file. +The helper gives every cell a unique output name and rejects missing artifacts. + +Keep prompt, negative prompt, seed, shape, steps, guidance, dtype, topology, +and residency fixed. Lossless comparisons require byte-identical artifacts. +For `quality=high`, report aggregate and worst-frame SSIM/PSNR; the repository +defaults are 0.95/28 dB for images and 0.92/24 dB for video unless the model's +checked-in consistency metadata defines a different threshold. A performance +PR needs repeated saved-request e2e improvement of at least 1.5%, a +representative profile, and before/after image or video evidence. + MiniMax-H3 is always an eager consistency case on current main. Use `--model minimax-h3-t2va`; its preset writes the H3 request fields through a -generated config and suppresses the helper's global compile default. +generated config and suppresses the helper's global compile default. Do not +turn the model's nominal BCG support gate into a performance claim: prompt- +dependent packed-sequence host boundaries can differ between warmup and the +serving request. A valid H3 BCG experiment must prove that every captured +segment replays, keeps the MP4 byte-identical, and does not trade latency for +the extra graph memory. For FLUX-family manual profiling runs with a quantized transformer override: - use `sglang generate` directly diff --git a/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-benchmark-profile/benchmark-and-profile.md b/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-benchmark-profile/benchmark-and-profile.md index 5d9b8ee2a..41ee2b06c 100644 --- a/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-benchmark-profile/benchmark-and-profile.md +++ b/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-benchmark-profile/benchmark-and-profile.md @@ -139,6 +139,58 @@ The helper defaults to eager. Add `--torch-compile` only for a labeled compile control. `--no-torch-compile` remains accepted for compatibility but is no longer required. +Run one explicit quality or BCG comparator with `--quality lossless|high` and +`--breakable-cuda-graph`. BCG and `torch.compile` are intentionally mutually +exclusive in this helper. A high+BCG command is only a compatibility probe: +it is invalid if request-scoped DiT fusions mount after the lossless warmup +graphs were captured. When a preset has explicit width and height, the helper +declares that same `--warmup-resolutions` value automatically: + +```bash +PYTHONPATH=python python3 "$BENCH_PY" \ + --model longcat-image \ + --quality high \ + --breakable-cuda-graph \ + --label bcg-high \ + --output-dir "${BENCH_DIR}" +``` + +For optimization discovery, use the full repeated matrix. It runs +Eager/BCG/BCG/Eager at `lossless`, then the same sequence at `high`, while +holding one GPU set and one isolated checkpoint cache. The high+BCG cells test +whether the combination is actually supported; do not average them when the +runtime rejects the combination or the helper detects a late quality-fusion +mount. The helper also hashes every generated artifact, first requires the two +Eager rows at each quality to agree, then rejects any BCG row whose hash differs +from that Eager reference. Cleanup occurs only after all eight runs, including +on failure or interruption: + +```bash +MODEL_CACHE_ROOT=/path/to/task-owned/model-caches +PYTHONPATH=python python3 "$BENCH_PY" \ + --model longcat-image \ + --quality-bcg-matrix \ + --label h200 \ + --output-dir "${BENCH_DIR}" \ + --model-cache-root "${MODEL_CACHE_ROOT}" \ + --cleanup-model-cache +``` + +Before starting, confirm the chosen GPU set has no foreign process and remains +unchanged through every run boundary. The helper rejects a BCG row unless its +log contains `[Diffusion BCG] captured` and contains none of: support-gate +disable, capture failure, `serving signature MISSED`, a message that no graph +will be captured, or a request-scoped high-quality DiT fusion mounted after +capture. Do not average rejected rows with valid results. + +BCG signatures include more than width and height. The public +`--warmup-resolutions` flag declares only `WxH`; synthetic warmup still uses +the model's own frame-count and conditioning defaults. A short video preset +can therefore capture a default temporal shape and miss the actual request +even at the same resolution. The helper marks that row invalid. Use a request +whose complete temporal/conditioning contract matches warmup, or fix the +model's BCG warmup/padding contract before claiming a speedup. + The helper sets `SGLANG_DIFFUSION_SYNC_STAGE_PROFILING=1` for accurate stage attribution. Set it to `0` explicitly only when collecting an e2e-only run and do not compare its per-stage values with synchronized results. @@ -237,12 +289,40 @@ Use the preset categories this way: | `ltx23-ti2v-two-stage` | `Lightricks/LTX-2.3` | Yes: `ltx2.3_twostage_ti2v_2gpus` | Nightly cat image, motion prompt, `LTX2TwoStagePipeline`, 2 GPUs, `--cfg-parallel-size 2`, 768x512, 121 frames, seed 42 | | `ideogram4-fp8` | `ideogram-ai/ideogram-4-fp8` | Yes: `ideogram4_fp8_t2i_2gpu` | Prompt, 1024x1024, seed 42, 2 GPUs, TP size 2, FlashAttention backend; sampling preset owns steps/guidance | | `cosmos3-super-t2v` | `nvidia/Cosmos3-Super` | Yes: `cosmos3_super_t2v_2gpu` | Prompt, 1280x720, 81 frames, seed 42, 2 GPUs, TP size 2, guardrails disabled for benchmark isolation | +| `cosmos3-super-t2v-cfg2tp2` | `nvidia/Cosmos3-Super` | No | Explicit four-GPU TP2 x CFG2 throughput comparator. On H200 it was 48.00% faster end to end than TP2, but the topology changed the deterministic output (SSIM 0.914244, PSNR 29.469771 dB), so do not treat it as lossless-equivalent or select it automatically. | | `wan-i2v` | `Wan-AI/Wan2.2-I2V-A14B-Diffusers` | Yes: `wan22_i2v_a14b_720p` | Nightly cat image and motion prompt, 1280x720, 81 frames, 4 GPUs, CFG parallel, Ulysses degree 2, text encoder CPU offload and pinned CPU memory | | `minimax-h3-t2va` | `MiniMaxAI/MiniMax-H3` | Yes: `minimax_h3_t2va_5s` | H3 FL2VA-partition T2VA baseline: 1344x768 resolved canvas, 5 seconds / 124 frames at 24 fps, 50 joint video-audio steps, 4 GPUs, TP2 + Ulysses2, eager BF16/FP32. The helper writes H3's request contract to a generated config. | | `longcat-image` | `meituan-longcat/LongCat-Image` | No | Eager DiT baseline at 1024x1024, 50 steps, guidance 4.5; prompt rewrite is disabled so Qwen2.5-VL does not contaminate the DiT A/B. | | `sana-video` | `Efficient-Large-Model/SANA-Video_2B_480p_diffusers` | No | CI-sized eager T2V baseline: 832x480, 17 frames, 8 steps, guidance 6.0. Compare `quality=lossless` and `quality=high`; high enables the BF16-input first linear-attention GEMM while retaining FP32 output and the FP32 second GEMM. | +| `sana-wm-bidirectional` | `Efficient-Large-Model/SANA-WM_bidirectional` | No | Dense two-stage TI2V baseline at the native 1280x704 shape, 49 frames, 16 fps, 20 steps, guidance 4.5, and a 48-frame forward/left action program. Uses the shared cat fixture. | +| `sana-wm-streaming` | `Efficient-Large-Model/SANA-WM_streaming` | No | Matching offline chunk-causal two-stage baseline with the streaming DiT and chunked refiner enabled; uses the same shape, fixture, seed, and camera action for comparison. | | `lingbot-video-moe` | `robbyant/lingbot-video-moe-30b-a3b` | No | One-GPU eager baseline using the CI structured-JSON caption, 384x640, 17 frames, 12 steps, and text-encoder CPU offload. | +| `lingbot-world` | `robbyant/lingbot-world-fast-diffusers` | No | One-H200 offline single-chunk profile for the registered causal DMD path: 832x480x9, four steps, guidance 1.0, the shared image fixture, and forward-camera actions for all nine frames. Keep stateful websocket latency as a separate metric. | +| `lingbot-world-v2` | `robbyant/lingbot-world-v2-14b-causal-fast-diffusers` | No | Matching controlled single-chunk profile for the separately registered v2 checkpoint. The fixed shape, action program, and schedule make v1/v2 hotspot comparisons reproducible without presenting one-chunk e2e as stateful realtime latency. | +| `fastwan21-t2v-1.3b` | `FastVideo/FastWan2.1-T2V-1.3B-Diffusers` | No | One-GPU 832x480, 61-frame, 3-step DMD baseline. The preset pins manual mode with a resident DiT so lossless/high comparisons do not measure an offload-policy change. | +| `wan21-t2v-1.3b` | `Wan-AI/Wan2.1-T2V-1.3B-Diffusers` | No | Registered one-GPU 832x480, 81-frame Wan2.1 baseline at 50 steps and guidance 3.0. Keep it separate from FastWan and TurboWan because the longer schedule changes the end-to-end weight of VAE optimizations. | +| `wan21-t2v-14b` | `Wan-AI/Wan2.1-T2V-14B-Diffusers` | No | Cookbook-aligned four-GPU CFG/Ulysses baseline at 832x480, 81 frames, 50 steps, and guidance 5.0. Text encoding stays CPU-offloaded as in the documented deployment command. | +| `wan21-i2v-14b-480p` | `Wan-AI/Wan2.1-I2V-14B-480P-Diffusers` | No | Four-GPU CFG/Ulysses image-conditioned baseline at 832x480, 81 frames, 50 steps, and guidance 5.0. Uses the shared cat fixture and its motion prompt. | +| `wan21-i2v-14b-720p` | `Wan-AI/Wan2.1-I2V-14B-720P-Diffusers` | No | Four-GPU CFG/Ulysses image-conditioned baseline at 1280x720, 81 frames, 50 steps, and guidance 5.0. Keep it separate from 480P because it is a distinct checkpoint and attention shape. | +| `wan21-fun-inp-1.3b` | `weizhou03/Wan2.1-Fun-1.3B-InP-Diffusers` | No | Registered one-GPU Wan2.1 Fun image-conditioned path at 832x480, 81 frames, 50 steps, and guidance 6.0. Uses the shared cat fixture and motion prompt. | +| `krea2-turbo` | `krea/Krea-2-Turbo` | No | Recent T2I checkpoint at 1024x1024, 8 steps, guidance 1.0. | +| `krea2-raw` | `krea/Krea-2-Raw` | No | Recent T2I checkpoint at 1024x1024, 50 steps, guidance 4.5; keep separate from Turbo because CFG and the longer schedule change the hotspot mix. | +| `ideogram4-fast` | `fal/ideogram-v4-fast` | No | Recent distilled T2I checkpoint at 1024x1024; the registered sampling class owns its step and guidance defaults. | +| `ideogram4-instant` | `fal/ideogram-v4-instant` | No | Recent distilled T2I checkpoint at 1024x1024; the registered sampling class owns its step and guidance defaults. | +| `longlive2-t2v` | `Rabinovich/LongLive-2.0-5B-Diffusers` | No | CI-aligned 832x480, 61-frame causal DMD T2V baseline at 4 steps and guidance 1.0. | +| `longlive2-i2v` | `Rabinovich/LongLive-2.0-5B-Diffusers` | No | CI-aligned 960x928, 61-frame causal DMD I2V baseline using the cat image. | +| `fast-hunyuan` | `FastVideo/FastHunyuan-diffusers` | No | Validated one-H200 832x480, 61-frame FastHunyuan baseline using its registered 6-step schedule. | +| `turbowan21-t2v-1.3b` | `IPostYellow/TurboWan2.1-T2V-1.3B-Diffusers` | No | Registered one-GPU TurboWan path at 832x480, 81 frames, and 4 steps. | +| `turbowan21-t2v-14b-480p` | `IPostYellow/TurboWan2.1-T2V-14B-Diffusers` | No | One-H200 TurboWan 14B path at 832x480, 81 frames, and its 4-step DMD schedule. | +| `turbowan21-t2v-14b-720p` | `IPostYellow/TurboWan2.1-T2V-14B-720P-Diffusers` | No | One-H200 high-resolution TurboWan 14B path at 1280x720, 81 frames, and its 4-step DMD schedule. Keep it separate because it is a distinct checkpoint. | +| `turbowan22-i2v-a14b` | `IPostYellow/TurboWan2.2-I2V-A14B-Diffusers` | No | Four-GPU CFG/Ulysses image-conditioned baseline at 1280x720, 81 frames, and its 4-step DMD schedule. Uses the shared cat fixture and keeps both high- and low-noise guidance at 3.5. | +| `helios-mid` | `BestWishYsh/Helios-Mid` | No | CI-sized 640x384, 33-frame pyramid-SR baseline using Helios-Mid's 20-step schedule. | +| `helios-distilled` | `BestWishYsh/Helios-Distilled` | No | CI-sized 640x384, 33-frame DMD baseline at 10 steps and guidance 1.0. | +| `joy-echo` | `jdopensource/JoyAI-Echo` | No | CI-aligned two-GPU Ulysses baseline at 640x384, 33 frames, 8 steps, with the cross-request memory bank disabled for isolated single-request timing. | | `cosmos3-edge-t2i` | `nvidia/Cosmos3-Edge` | No | One-GPU eager T2I baseline at Edge's native 640x640 shape, 35 steps, guidance 7.0. | +| `cosmos3-edge-t2v` | `nvidia/Cosmos3-Edge` | No | One-GPU eager T2V baseline at Edge's native 832x480 video shape, 81 frames, 35 steps, and guidance 5.0. | +| `cosmos3-edge-i2v` | `nvidia/Cosmos3-Edge` | No | Matching one-GPU I2V baseline with the shared cat fixture; keep it separate because image conditioning adds the VAE encode and latent-mask paths. | +| `cosmos3-super-i2v` | `nvidia/Cosmos3-Super-Image2Video` | No | Registered specialized I2V checkpoint with the shared cat fixture; 1280x720, 81 frames, 35 steps, guidance 6.0, flow shift 10.0, seed 42, 2 GPUs, TP size 2, and guardrails disabled for benchmark isolation. | | `cosmos3-super-t2i-distilled` | `nvidia/Cosmos3-Super-Text2Image-4Step` | No | Four-GPU eager distilled T2I baseline. The checkpoint owns its fixed sigma schedule; the preset does not override the step count. | | `ltx25` | `Lightricks/LTX-2.5-Diffusers` | No | One-stage distilled eager baseline at 960x544, 121 frames, 8 steps, guidance 1.0. | | `ltx25-diffusion-decoder` | `Lightricks/LTX-2.5-Diffusers` | No | Same fixed DiT workload with `--use-diffusion-decoder`; attribute decoder time separately and confirm NATTEN `na3d` is active. | @@ -258,12 +338,14 @@ Use the preset categories this way: | `glm-image` | `zai-org/GLM-Image` | No | Current-source extra for GLM-Image | | `sana-1.5-1.6b` | `Efficient-Large-Model/SANA1.5_1.6B_1024px_diffusers` | No | Current-source extra for a SANA native image path | | `fastwan22-ti2v-5b` | `FastVideo/FastWan2.2-TI2V-5B-FullAttn-Diffusers` | No | Current-source extra matching the FastWan2.2 TI2V registered path | +| `wan22-t2v-nvfp4` | `nvidia/Wan2.2-T2V-A14B-Diffusers-NVFP4` | No | Blackwell-only one-GPU ModelOpt NVFP4 T2V baseline at 832x480 and 81 frames. Manual mode keeps the DiT resident so the trace measures FP4 kernels instead of layerwise transfer. | | `ltx23-hq-two-stage` | `Lightricks/LTX-2.3` | No | Current-source extra for `LTX2TwoStageHQPipeline` with `--ltx2-two-stage-device-mode=original`; high-resolution and VRAM-heavy | | `ltx23-one-stage` | `Lightricks/LTX-2.3` | No | Skill-only extra preset for the native `LTX-2.3` one-stage baseline; 2 GPUs, 768x512, 121 frames, fps 24, 30 steps, guidance 3.0, seed 1234 | | `ltx23-two-stage` | `Lightricks/LTX-2.3` | No | Skill-only high-resolution stress preset for the native `LTX-2.3` two-stage path; uses `LTX2TwoStagePipeline`, 2 GPUs, 1536x1024, 121 frames, fps 24, 30 steps, guidance 3.0, seed 1234 | | `ltx23-two-stage-cfg-parallel` | `Lightricks/LTX-2.3` | No | Skill-only high-resolution CFG-parallel stress preset matching `ltx23-two-stage` plus `--cfg-parallel-size 2` | -| `hunyuanvideo` | `hunyuanvideo-community/HunyuanVideo` | No | Skill-only extra preset | -| `mova-720p` | `OpenMOSS-Team/MOVA-720p` | No | Skill-only extra preset | +| `hunyuanvideo` | `hunyuanvideo-community/HunyuanVideo` | No | Skill-only native T2V preset at a model-supported 960x544 resolution, 65 requested frames, and 30 steps. Sequence-parallel runs may increase the frame count to satisfy their topology; record the resolved shape from the runtime log. | +| `mova-360p` | `OpenMOSS-Team/MOVA-360p` | No | Two-GPU Ulysses I2VA baseline at 640x352 and 193 frames. Uses the upstream single-person fixture and a two-step profiling schedule. | +| `mova-720p` | `OpenMOSS-Team/MOVA-720p` | No | Four-GPU Ulysses I2VA baseline at 1280x720 and 193 frames. Uses the same upstream single-person fixture and two-step profiling schedule. | | `helios` | `BestWishYsh/Helios-Base` | No | Skill-only extra preset | | `joyai-edit` | `jdopensource/JoyAI-Image-Edit-Diffusers` | No | Skill-only JoyAI image-edit preset; uses the cat image, 1024x1024, 40 steps, guidance 4.0, 2-GPU CFG parallel | | `firered-edit-1.0` | `FireRedTeam/FireRed-Image-Edit-1.0` | No | Skill-only FireRed 1.0 image-edit preset; QwenImageEditPlus native path; uses 2-GPU CFG parallel | @@ -282,6 +364,11 @@ For MiniMax-H3, keep the native contract intact: `--model-variant`; do not point at a checkpoint subdirectory - use eager BF16/FP32 for consistency ground truth; current H3 `torch.compile` changes numerical output +- keep BCG off in the validated recipe. The support gate alone is not enough: + prompt-dependent packed-sequence host boundaries can differ between warmup + and serving and cause a signature miss. Any experimental fix must prove real + segment replay, byte-identical media, and an e2e win without excessive graph + memory - use Ulysses, not Ring, for H3's packed multi-segment attention; CFG parallel is invalid because the released pipeline has one denoising branch - keep the released overlapping tiled video-VAE decode. H3 rejects @@ -531,7 +618,24 @@ Always keep: - exact command line, model shape, dtype, request `quality`, GPU topology, and whether synchronized stage profiling was enabled -Never keep a perf dump produced after a diffusers-backend fallback. +Never keep a perf dump produced after a diffusers-backend fallback. Also reject +a zero-exit run if either the requested perf dump or generated media is absent: +some generation failures are reported through the response payload without a +nonzero process exit. + +For `quality=lossless`, compare saved artifact hashes and require byte equality +for a claimed lossless fast path or BCG change. For `quality=high`, keep the +lossless artifact as ground truth and report both aggregate and worst-frame +SSIM/PSNR. Repository defaults are SSIM 0.95 / PSNR 28 dB for images and SSIM +0.92 / PSNR 24 dB for videos; checked-in model/hardware consistency metadata +may override them. Always inspect the image or a start/middle/end video contact +sheet in addition to scalar metrics. + +Use denoise timing to locate the opportunity, but gate a performance PR on +repeated saved-request end-to-end time. The project threshold for this sweep is +at least 1.5% mean e2e improvement on same-GPU ABBA runs. Attach one +representative baseline/candidate profile plus before/after images or videos to +the PR description. Stage durations are host wall times around asynchronous GPU launches unless `SGLANG_DIFFUSION_SYNC_STAGE_PROFILING=1`. Without the sync, queued denoise @@ -672,10 +776,14 @@ This skill intentionally stops here. It tells you whether you are looking at: - [ ] fixed-shape baseline perf dump saved - [ ] fixed-shape new perf dump saved -- [ ] request `quality` and `SGLANG_DIFFUSION_SYNC_STAGE_PROFILING` match +- [ ] quality/BCG applicability matrix attempted on one GPU set +- [ ] BCG rows show capture and no disable/failure/signature-miss/late-quality-fusion marker +- [ ] request shape, seed, steps, guidance, topology, residency, and synchronized stage profiling match - [ ] `compare_perf.py` table generated - [ ] one representative `torch.profiler` trace saved - [ ] hotspot classified against `existing-fast-paths.md` -- [ ] reference image or video checked for correctness +- [ ] lossless artifact hash is exact; high-quality aggregate and worst-frame SSIM/PSNR pass the checked-in threshold +- [ ] reference image or start/middle/end video contact sheet checked visually +- [ ] any PR claim has repeated saved-request e2e improvement >= 1.5% - [ ] task-owned checkpoint cache cleaned and ledger shows zero residual weight files - [ ] any remaining kernel work handed off with perf/profile evidence attached diff --git a/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-benchmark-profile/existing-fast-paths.md b/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-benchmark-profile/existing-fast-paths.md index ec7b8a74e..14ca25968 100644 --- a/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-benchmark-profile/existing-fast-paths.md +++ b/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-benchmark-profile/existing-fast-paths.md @@ -118,8 +118,10 @@ framework-specific optimization workflow. fusions and decode-scoped VAE rewrites. Mounting is all-or-nothing per transformer/fusion family; VAE gates reset after every decode. - Current families include FLUX affine-folded LN+modulate / fused GELU sites, - GLM-Image fused GELU sites, generic KL VAE decoder rewrites used by - FLUX.1/FLUX.2/Z-Image/SD3, and Wan VAE RMSNorm+SiLU. + GLM/Qwen/Hunyuan/LTX fused GELU sites, LTX RMSNorm+modulate, Hunyuan QK + RMSNorm, Ideogram gated RMSNorm, SANA-Video linear attention, generic KL VAE + decoder rewrites used by FLUX.1/FLUX.2/Z-Image/SD3, and Wan VAE + RMSNorm+SiLU. - Do not confuse request `--quality` with `--output-quality`, which controls output-file compression rather than model math. - Validation: `test_quality_gate.py`, `test_fused_ln_modulate.py`, @@ -295,6 +297,20 @@ framework-specific optimization workflow. - Scope: this is a mainline SANA model fast path. Query projection in cross-attention remains separate because it uses denoising hidden states, while K/V share step-invariant encoder hidden states. - Workflow rule: if a SANA trace shows separate self-attention `to_q`, `to_k`, `to_v` GEMMs, or separate cross-attention `to_k` and `to_v` GEMMs, treat that as a regressed existing packed-projection path before proposing a new GEMM fusion. +**Request-Scoped DiT Fusions with Breakable CUDA Graphs** + +- `quality=high` DiT sites are mounted at a request boundary. BCG warmup uses + the model's lossless sampling default unless a quality-aware graph variant + was captured explicitly. +- A graph captured before the high-quality mount retains the lossless module + branches. Replaying it after the mount silently bypasses the requested high + kernels even when the tensor signature matches. +- Workflow rule: a high+BCG cell is valid only when the model has no + request-scoped DiT quality sites, or when logs prove those sites were mounted + before the matching graph capture. A mount after `[Diffusion BCG] captured` + invalidates the row; do not use its latency or output as high-quality + evidence. + **Recent Model Audit Boundaries** - LongCat-Image supports breakable CUDA graph at fixed, captured resolutions. @@ -318,7 +334,10 @@ framework-specific optimization workflow. - LingBot Video MoE's router implements sigmoid+bias grouped top-k in `multimodal_gen/runtime/layers/moe.py`. Check parameter and output-order compatibility with `srt/layers/moe/topk.py::biased_grouped_topk` before - writing a new router kernel. + writing a new router kernel. Its released eager path still expands RMSNorm + into `pow/mean/rsqrt` chains. #35969 is a measured `quality=high` candidate + that dispatches existing Triton row kernels by weight dtype and hidden size; + it is not current-main behavior until the PR merges. - LTX-2.5 reuses the mature LTX-2 DiT paths. Treat the optional diffusion decoder separately: confirm NATTEN `na3d` is active, then inspect its per-block 3D RoPE construction and split QKV/SwiGLU projections. @@ -381,6 +400,13 @@ framework-specific optimization workflow. every additional served resolution in `--warmup-resolutions`, and use `--bcg-text-buckets` for prompt signatures. Check this path before proposing a second graph-capture mechanism for launch-bound traces. +- A valid BCG benchmark must show `[Diffusion BCG] captured` and no support + disable, capture failure, `serving signature MISSED`, or eager-fallback + marker. Width and height are not the whole signature: public + `--warmup-resolutions` does not override a video model's synthetic warmup + frame count, so a short profiling request can capture the default temporal + shape and then miss during serving. Reject that timing instead of labeling + it BCG. - LongCat-Image uses this generic runner directly: one 1024x1024 capture covers short and long prompts because text conditioning is fixed at 512 tokens. Keep eager as the baseline because the gain is hardware-dependent; an H200 @@ -410,7 +436,10 @@ relying on any file path, flag, or claim about whether the work has merged. - #29361 LTX2 residual-gate CUDA fast path for `residual + update * gate`. - #34172 LTX2 quality-high fusion; #34305/#34314 Ideogram eager fusions. - #34584 Wan TI2V modulation/RoPE; #34616 FLUX2; #34617 Hunyuan; - #34619 GLM; #34620 ERNIE; #34928 SANA; #34932 Cosmos3. + #34619 GLM; #34620 ERNIE; #34928 SANA; #34932 Cosmos3; #35728 + SANA-Video linear attention. + - #35961 SANA-Video lossless shared-kernel reuse and #35969 LingBot + `quality=high` RMSNorm are open candidates, not current-main fast paths. - VAE and decode-side acceleration: - #22531 LTX2 parallel VAE support and #20927 batched tiled VAE decode (draft). - Attention, communication, and runtime scheduling: @@ -426,7 +455,8 @@ relying on any file path, flag, or claim about whether the work has merged. - #19516 Qwen-Image CUDA Graph. - #21912 Z-Image Turbo FP8 full quantization and CUDA Graph. - #34174 automatic default-resolution BCG warmup; #34210 Z-Image BCG - correctness; #34929 LTX2.3 BCG. #34618 is a closed Cosmos BCG experiment, + correctness; #34929 LTX2.3 BCG; #35724 LongCat-Image BCG; #35729 + SANA-Video fixed-300-token BCG. #34618 is a closed Cosmos BCG experiment, not a reusable mainline fast path. **Constraints and Fallbacks** diff --git a/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-benchmark-profile/scripts/bench_diffusion_denoise.py b/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-benchmark-profile/scripts/bench_diffusion_denoise.py index 382ae44e2..fb710cc1e 100755 --- a/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-benchmark-profile/scripts/bench_diffusion_denoise.py +++ b/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-benchmark-profile/scripts/bench_diffusion_denoise.py @@ -16,6 +16,10 @@ Usage: # Opt in to a compile control (presets are eager by default) python3 python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-benchmark-profile/scripts/bench_diffusion_denoise.py --model flux --torch-compile + # Check Eager/BCG at lossless/high on one GPU set; high+BCG is invalid when + # request-scoped DiT fusions mount only after lossless graph capture. + python3 python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-benchmark-profile/scripts/bench_diffusion_denoise.py --model sana-video --quality-bcg-matrix --model-cache-root /task/model-caches --cleanup-model-cache + # Clean an isolated model cache even if the run fails or is interrupted python3 python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-benchmark-profile/scripts/bench_diffusion_denoise.py --model longcat-image --model-cache-root /task/model-caches --cleanup-model-cache @@ -37,6 +41,7 @@ Input images required for image-guided models: """ import argparse +import hashlib import json import os import shlex @@ -74,8 +79,38 @@ DIFFUSERS_FALLBACK_SIGNALS = ( "using diffusers backend", "loaded diffusers pipeline", ) +BENCHMARK_QUALITY_LEVELS = ("lossless", "high") +BCG_CAPTURE_SIGNAL = "[diffusion bcg] captured" +BCG_INVALID_SIGNALS = ( + "[diffusion bcg] capture failed", + "[diffusion bcg] disabled", + "[diffusion bcg] serving signature missed", + "no graph will be captured", + "quality='high' cannot be used with breakable cuda graphs", +) +BCG_LATE_QUALITY_FUSION_SIGNAL = "quality fusion mounted after BCG capture" +QUALITY_BCG_ABBA_MATRIX = ( + ("eager-lossless-a", "lossless", False), + ("bcg-lossless-a", "lossless", True), + ("bcg-lossless-b", "lossless", True), + ("eager-lossless-b", "lossless", False), + ("eager-high-a", "high", False), + ("bcg-high-a", "high", True), + ("bcg-high-b", "high", True), + ("eager-high-b", "high", False), +) + + +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as file: + for chunk in iter(lambda: file.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + CATALOG_TABLE_WIDTH = 140 -RESULTS_TABLE_WIDTH = 105 +RESULTS_TABLE_WIDTH = 124 MODEL_CACHE_MARKER = ".sglang-diffusion-benchmark-cache" MODEL_WEIGHT_SUFFIXES = { ".bin", @@ -85,6 +120,7 @@ MODEL_WEIGHT_SUFFIXES = { ".pth", ".safetensors", } +GENERATED_OUTPUT_SUFFIXES = {".jpeg", ".jpg", ".mp4", ".png", ".wav", ".webp"} NIGHTLY_PRESET_ORDER = ( "flux", "flux2", @@ -150,6 +186,9 @@ LINGBOT_VIDEO_PROMPT = json.dumps( }, separators=(",", ":"), ) +LINGBOT_WORLD_CONFIG_OVERRIDES = { + "actions": [["w"] for _ in range(9)], +} # --------------------------------------------------------------------------- # Model configs — kept in exact sync with benchmark-and-profile.md @@ -296,6 +335,22 @@ MODELS = { "--tp-size=2", ], }, + # Explicit throughput comparator. CFG parallelism changes sampling numerics, + # so compare its output against the TP=2 preset before using the speedup. + "cosmos3-super-t2v-cfg2tp2": { + "path": "nvidia/Cosmos3-Super", + "prompt": "A cat and a dog baking a cake together in a kitchen.", + "env": { + "SGLANG_DISABLE_COSMOS3_GUARDRAILS": "1", + }, + "extra_args": [ + "--width=1280", + "--height=720", + "--num-frames=81", + "--num-gpus=4", + "--tp-size=2", + ], + }, # 11. Nightly: wan22_i2v_a14b_720p # Requires: /inputs/diffusion_benchmark/figs/cat.png "wan-i2v": { @@ -381,6 +436,42 @@ MODELS = { "--performance-mode=manual", ], }, + # Requires: /inputs/diffusion_benchmark/figs/cat.png + "sana-wm-bidirectional": { + "path": "Efficient-Large-Model/SANA-WM_bidirectional", + "prompt": "a camera moving forward and turning left", + "image_path": str(ASSET_DIR / "cat.png"), + "seed": 42, + "extra_args": [ + "--pipeline-class-name=SanaWMTwoStagePipeline", + "--width=1280", + "--height=704", + "--num-frames=49", + "--fps=16", + "--num-inference-steps=20", + "--guidance-scale=4.5", + "--action=w-16,wl-16,l-16", + "--performance-mode=manual", + ], + }, + # Requires: /inputs/diffusion_benchmark/figs/cat.png + "sana-wm-streaming": { + "path": "Efficient-Large-Model/SANA-WM_streaming", + "prompt": "a camera moving forward and turning left", + "image_path": str(ASSET_DIR / "cat.png"), + "seed": 42, + "extra_args": [ + "--pipeline-class-name=SanaWMTwoStagePipeline", + "--streaming", + "--refiner-chunked", + "--width=1280", + "--height=704", + "--num-frames=49", + "--fps=16", + "--action=w-16,wl-16,l-16", + "--performance-mode=manual", + ], + }, "lingbot-video-moe": { "path": "robbyant/lingbot-video-moe-30b-a3b", "prompt": LINGBOT_VIDEO_PROMPT, @@ -395,6 +486,290 @@ MODELS = { "--performance-mode=manual", ], }, + # Requires: /inputs/diffusion_benchmark/figs/cat.png + "lingbot-world": { + "path": "robbyant/lingbot-world-fast-diffusers", + "prompt": "A slow aerial orbit around a pastel island hotel in the ocean.", + "image_path": str(ASSET_DIR / "cat.png"), + "seed": 42, + "config_overrides": LINGBOT_WORLD_CONFIG_OVERRIDES, + "extra_args": [ + "--width=832", + "--height=480", + "--num-frames=9", + "--fps=16", + "--num-inference-steps=4", + "--guidance-scale=1.0", + "--text-encoder-cpu-offload", + "--warmup-mode=off", + ], + }, + # Requires: /inputs/diffusion_benchmark/figs/cat.png + "lingbot-world-v2": { + "path": "robbyant/lingbot-world-v2-14b-causal-fast-diffusers", + "prompt": "A slow aerial orbit around a pastel island hotel in the ocean.", + "image_path": str(ASSET_DIR / "cat.png"), + "seed": 42, + "config_overrides": LINGBOT_WORLD_CONFIG_OVERRIDES, + "extra_args": [ + "--width=832", + "--height=480", + "--num-frames=9", + "--fps=16", + "--num-inference-steps=4", + "--guidance-scale=1.0", + "--text-encoder-cpu-offload", + "--warmup-mode=off", + ], + }, + "fastwan21-t2v-1.3b": { + "path": "FastVideo/FastWan2.1-T2V-1.3B-Diffusers", + "prompt": "A curious raccoon walks through a sunlit forest.", + "extra_args": [ + "--width=832", + "--height=480", + "--num-frames=61", + "--fps=16", + "--num-inference-steps=3", + "--performance-mode=manual", + "--dit-layerwise-offload=false", + "--dit-cpu-offload=false", + ], + }, + "wan21-t2v-1.3b": { + "path": "Wan-AI/Wan2.1-T2V-1.3B-Diffusers", + "prompt": "A curious raccoon walks through a sunlit forest.", + "extra_args": [ + "--width=832", + "--height=480", + "--num-frames=81", + "--fps=16", + "--num-inference-steps=50", + "--guidance-scale=3.0", + ], + }, + "wan21-t2v-14b": { + "path": "Wan-AI/Wan2.1-T2V-14B-Diffusers", + "prompt": "A curious raccoon", + "extra_args": [ + "--width=832", + "--height=480", + "--num-frames=81", + "--fps=16", + "--num-inference-steps=50", + "--guidance-scale=5.0", + "--num-gpus=4", + "--enable-cfg-parallel", + "--ulysses-degree=2", + "--text-encoder-cpu-offload", + "--pin-cpu-memory", + ], + }, + "wan21-i2v-14b-480p": { + "path": "Wan-AI/Wan2.1-I2V-14B-480P-Diffusers", + "prompt": "The cat starts walking slowly towards the camera.", + "image_path": str(ASSET_DIR / "cat.png"), + "extra_args": [ + "--width=832", + "--height=480", + "--num-frames=81", + "--fps=16", + "--num-inference-steps=50", + "--guidance-scale=5.0", + "--num-gpus=4", + "--enable-cfg-parallel", + "--ulysses-degree=2", + "--text-encoder-cpu-offload", + "--pin-cpu-memory", + ], + }, + "wan21-i2v-14b-720p": { + "path": "Wan-AI/Wan2.1-I2V-14B-720P-Diffusers", + "prompt": "The cat starts walking slowly towards the camera.", + "image_path": str(ASSET_DIR / "cat.png"), + "extra_args": [ + "--width=1280", + "--height=720", + "--num-frames=81", + "--fps=16", + "--num-inference-steps=50", + "--guidance-scale=5.0", + "--num-gpus=4", + "--enable-cfg-parallel", + "--ulysses-degree=2", + "--text-encoder-cpu-offload", + "--pin-cpu-memory", + ], + }, + "wan21-fun-inp-1.3b": { + "path": "weizhou03/Wan2.1-Fun-1.3B-InP-Diffusers", + "prompt": "The cat starts walking slowly towards the camera.", + "image_path": str(ASSET_DIR / "cat.png"), + "extra_args": [ + "--width=832", + "--height=480", + "--num-frames=81", + "--fps=16", + "--num-inference-steps=50", + "--guidance-scale=6.0", + ], + }, + "krea2-turbo": { + "path": "krea/Krea-2-Turbo", + "prompt": "A red fox sitting in fresh snow, golden hour, photorealistic.", + "extra_args": [ + "--width=1024", + "--height=1024", + "--num-inference-steps=8", + "--guidance-scale=1.0", + ], + }, + "krea2-raw": { + "path": "krea/Krea-2-Raw", + "prompt": "A red fox sitting in fresh snow, golden hour, photorealistic.", + "extra_args": [ + "--width=1024", + "--height=1024", + "--num-inference-steps=50", + "--guidance-scale=4.5", + ], + }, + "ideogram4-fast": { + "path": "fal/ideogram-v4-fast", + "prompt": "A vintage travel poster for Kyoto with crisp readable lettering.", + "extra_args": [ + "--width=1024", + "--height=1024", + ], + }, + "ideogram4-instant": { + "path": "fal/ideogram-v4-instant", + "prompt": "A vintage travel poster for Kyoto with crisp readable lettering.", + "extra_args": [ + "--width=1024", + "--height=1024", + ], + }, + "longlive2-t2v": { + "path": "Rabinovich/LongLive-2.0-5B-Diffusers", + "prompt": "A curious raccoon", + "extra_args": [ + "--width=832", + "--height=480", + "--num-frames=61", + "--num-inference-steps=4", + "--guidance-scale=1.0", + ], + }, + # Requires: /inputs/diffusion_benchmark/figs/cat.png + "longlive2-i2v": { + "path": "Rabinovich/LongLive-2.0-5B-Diffusers", + "prompt": "The cat starts walking slowly towards the camera.", + "image_path": str(ASSET_DIR / "cat.png"), + "extra_args": [ + "--width=960", + "--height=928", + "--num-frames=61", + "--num-inference-steps=4", + "--guidance-scale=1.0", + ], + }, + "fast-hunyuan": { + "path": "FastVideo/FastHunyuan-diffusers", + "prompt": "A curious raccoon", + "extra_args": [ + "--width=832", + "--height=480", + "--num-frames=61", + "--num-inference-steps=6", + ], + }, + "turbowan21-t2v-1.3b": { + "path": "IPostYellow/TurboWan2.1-T2V-1.3B-Diffusers", + "prompt": "A curious raccoon", + "extra_args": [ + "--width=832", + "--height=480", + "--num-frames=81", + "--num-inference-steps=4", + ], + }, + "turbowan21-t2v-14b-480p": { + "path": "IPostYellow/TurboWan2.1-T2V-14B-Diffusers", + "prompt": "A curious raccoon", + "extra_args": [ + "--width=832", + "--height=480", + "--num-frames=81", + "--num-inference-steps=4", + ], + }, + "turbowan21-t2v-14b-720p": { + "path": "IPostYellow/TurboWan2.1-T2V-14B-720P-Diffusers", + "prompt": "A curious raccoon", + "extra_args": [ + "--width=1280", + "--height=720", + "--num-frames=81", + "--num-inference-steps=4", + ], + }, + "turbowan22-i2v-a14b": { + "path": "IPostYellow/TurboWan2.2-I2V-A14B-Diffusers", + "prompt": "The cat starts walking slowly towards the camera.", + "image_path": str(ASSET_DIR / "cat.png"), + "extra_args": [ + "--width=1280", + "--height=720", + "--num-frames=81", + "--fps=16", + "--num-inference-steps=4", + "--guidance-scale=3.5", + "--guidance-scale-2=3.5", + "--num-gpus=4", + "--enable-cfg-parallel", + "--ulysses-degree=2", + "--text-encoder-cpu-offload", + "--pin-cpu-memory", + ], + }, + "helios-mid": { + "path": "BestWishYsh/Helios-Mid", + "prompt": "A curious raccoon", + "extra_args": [ + "--width=640", + "--height=384", + "--num-frames=33", + "--num-inference-steps=20", + ], + }, + "helios-distilled": { + "path": "BestWishYsh/Helios-Distilled", + "prompt": "A curious raccoon", + "extra_args": [ + "--width=640", + "--height=384", + "--num-frames=33", + "--num-inference-steps=10", + "--guidance-scale=1.0", + ], + }, + "joy-echo": { + "path": "jdopensource/JoyAI-Echo", + "prompt": "A curious raccoon", + "seed": 42, + "config_overrides": { + "enable_memory_bank": False, + }, + "extra_args": [ + "--width=640", + "--height=384", + "--num-frames=33", + "--num-inference-steps=8", + "--num-gpus=2", + "--ulysses-degree=2", + ], + }, "cosmos3-edge-t2i": { "path": "nvidia/Cosmos3-Edge", "prompt": "A warehouse robot folds a blue cloth on a clean workbench.", @@ -411,6 +786,62 @@ MODELS = { "--performance-mode=manual", ], }, + "cosmos3-edge-t2v": { + "path": "nvidia/Cosmos3-Edge", + "prompt": "A warehouse robot carefully places a blue box on a shelf.", + "seed": 42, + "env": { + "SGLANG_DISABLE_COSMOS3_GUARDRAILS": "1", + }, + "extra_args": [ + "--width=832", + "--height=480", + "--num-frames=81", + "--fps=24", + "--num-inference-steps=35", + "--guidance-scale=5.0", + "--performance-mode=manual", + ], + }, + "cosmos3-edge-i2v": { + "path": "nvidia/Cosmos3-Edge", + "prompt": "The cat starts walking slowly towards the camera.", + "image_path": str(ASSET_DIR / "cat.png"), + "seed": 42, + "env": { + "SGLANG_DISABLE_COSMOS3_GUARDRAILS": "1", + }, + "extra_args": [ + "--width=832", + "--height=480", + "--num-frames=81", + "--fps=24", + "--num-inference-steps=35", + "--guidance-scale=5.0", + "--performance-mode=manual", + ], + }, + # Requires: /inputs/diffusion_benchmark/figs/cat.png + "cosmos3-super-i2v": { + "path": "nvidia/Cosmos3-Super-Image2Video", + "prompt": "The cat starts walking slowly towards the camera.", + "image_path": str(ASSET_DIR / "cat.png"), + "seed": 42, + "env": { + "SGLANG_DISABLE_COSMOS3_GUARDRAILS": "1", + }, + "extra_args": [ + "--width=1280", + "--height=720", + "--num-frames=81", + "--fps=24", + "--num-inference-steps=35", + "--guidance-scale=6.0", + "--flow-shift=10.0", + "--num-gpus=2", + "--tp-size=2", + ], + }, "cosmos3-super-t2i-distilled": { "path": "nvidia/Cosmos3-Super-Text2Image-4Step", "prompt": "A warehouse robot folds a blue cloth on a clean workbench.", @@ -575,6 +1006,19 @@ MODELS = { "--num-frames=81", ], }, + # Blackwell-only ModelOpt NVFP4 comparator. + "wan22-t2v-nvfp4": { + "path": "nvidia/Wan2.2-T2V-A14B-Diffusers-NVFP4", + "prompt": "A cat and a dog baking a cake together in a kitchen.", + "extra_args": [ + "--width=832", + "--height=480", + "--num-frames=81", + "--performance-mode=manual", + "--dit-layerwise-offload=false", + "--dit-cpu-offload=false", + ], + }, "ltx23-hq-two-stage": { "path": "Lightricks/LTX-2.3", "prompt": "A beautiful sunset over the ocean", @@ -648,13 +1092,26 @@ MODELS = { "--text-encoder-cpu-offload", "--pin-cpu-memory", "--num-frames=65", - "--width=848", - "--height=480", + "--width=960", + "--height=544", "--num-inference-steps=30", ], }, - # Skill-only extra preset - # Requires: /inputs/diffusion_benchmark/figs/mova_single_person.jpg + # Skill-only extra presets + # Require: /inputs/diffusion_benchmark/figs/mova_single_person.jpg + "mova-360p": { + "path": "OpenMOSS-Team/MOVA-360p", + "prompt": 'A man in a blue blazer and glasses speaks in a formal indoor setting, framed by wooden furniture and a filled bookshelf. Quiet room acoustics underscore his measured tone as he delivers his remarks. At one point, he says, "I would also believe that this advance in AI recently was not unexpected."', + "image_path": str(ASSET_DIR / "mova_single_person.jpg"), + "extra_args": [ + "--adjust-frames=false", + "--num-gpus=2", + "--ulysses-degree=2", + "--num-frames=193", + "--fps=24", + "--num-inference-steps=2", + ], + }, "mova-720p": { "path": "OpenMOSS-Team/MOVA-720p", "prompt": 'A man in a blue blazer and glasses speaks in a formal indoor setting, framed by wooden furniture and a filled bookshelf. Quiet room acoustics underscore his measured tone as he delivers his remarks. At one point, he says, "I would also believe that this advance in AI recently was not unexpected."', @@ -1078,6 +1535,9 @@ def build_sglang_cmd( perf_dump_path: str | None = None, warmup: bool = True, torch_compile: bool = False, + quality: str = "lossless", + breakable_cuda_graph: bool = False, + bcg_text_buckets: list[int] | None = None, seed: int = 42, save_output: bool = True, artifact_dir: Path | None = None, @@ -1086,6 +1546,18 @@ def build_sglang_cmd( Build the `sglang generate` command for the given model. Matches the commands in benchmark-and-profile.md exactly. """ + if quality not in BENCHMARK_QUALITY_LEVELS: + raise ValueError( + f"quality must be one of {BENCHMARK_QUALITY_LEVELS}, got {quality!r}" + ) + if torch_compile and breakable_cuda_graph: + raise ValueError("torch.compile and breakable CUDA graph are comparators") + if bcg_text_buckets is not None: + if not breakable_cuda_graph: + raise ValueError("bcg_text_buckets requires breakable_cuda_graph=True") + if not bcg_text_buckets or any(bucket <= 0 for bucket in bcg_text_buckets): + raise ValueError("bcg_text_buckets must contain positive integers") + cfg = MODELS[model_key] cmd = [ @@ -1119,11 +1591,29 @@ def build_sglang_cmd( cmd.append(f"--config={config_path}") cmd.extend(cfg["extra_args"]) + cmd.append(f"--quality={quality}") if save_output: cmd.append("--save-output") if warmup: cmd.extend(["--warmup-mode", "request"]) + if breakable_cuda_graph: + cmd.append("--enable-breakable-cuda-graph") + parsed_args = _parse_cli_args(cmd) + if ( + "warmup-resolutions" not in parsed_args + and "width" in parsed_args + and "height" in parsed_args + ): + cmd.extend( + [ + "--warmup-resolutions", + f"{parsed_args['width']}x{parsed_args['height']}", + ] + ) + if bcg_text_buckets is not None: + cmd.append("--bcg-text-buckets") + cmd.extend(str(bucket) for bucket in bcg_text_buckets) if torch_compile and not cfg.get("force_eager", False): cmd.append("--enable-torch-compile") if perf_dump_path: @@ -1138,7 +1628,11 @@ def _run_benchmark_once_impl( output_dir: Path, warmup: bool = True, torch_compile: bool = False, + quality: str = "lossless", + breakable_cuda_graph: bool = False, + bcg_text_buckets: list[int] | None = None, model_cache_dir: Path | None = None, + cuda_visible_devices: str | None = None, ) -> dict: """Run a single benchmark pass and return results dict.""" perf_path = output_dir / f"{model_key}_{label}.json" @@ -1148,8 +1642,15 @@ def _run_benchmark_once_impl( perf_dump_path=str(perf_path), warmup=warmup, torch_compile=torch_compile, + quality=quality, + breakable_cuda_graph=breakable_cuda_graph, + bcg_text_buckets=bcg_text_buckets, artifact_dir=output_dir, ) + output_file_name = f"{model_key}-{label}" + cmd.extend( + ["--output-path", str(output_dir), "--output-file-name", output_file_name] + ) env = os.environ.copy() env.setdefault("FLASHINFER_DISABLE_VERSION_CHECK", "1") @@ -1178,7 +1679,9 @@ def _run_benchmark_once_impl( print(" fail early and report a misleading unsupported-model error.") return {"model": model_key, "label": label, "error": True, "elapsed_s": 0.0} - if not env.get("CUDA_VISIBLE_DEVICES"): + if cuda_visible_devices is not None: + env["CUDA_VISIBLE_DEVICES"] = cuda_visible_devices + elif not env.get("CUDA_VISIBLE_DEVICES"): env["CUDA_VISIBLE_DEVICES"] = ",".join( str(index) for index in pick_idle_gpus(required_gpus_for_model(model_key)) ) @@ -1199,12 +1702,28 @@ def _run_benchmark_once_impl( bufsize=1, ) fallback_detected = False + bcg_capture_detected = False + bcg_invalid_signals: set[str] = set() assert process.stdout is not None try: for line in process.stdout: print(line, end="") - if any(signal in line.lower() for signal in DIFFUSERS_FALLBACK_SIGNALS): + lower_line = line.lower() + if any(signal in lower_line for signal in DIFFUSERS_FALLBACK_SIGNALS): fallback_detected = True + if BCG_CAPTURE_SIGNAL in lower_line: + bcg_capture_detected = True + if ( + quality == "high" + and breakable_cuda_graph + and bcg_capture_detected + and "mounted " in lower_line + and "for quality=high" in lower_line + ): + bcg_invalid_signals.add(BCG_LATE_QUALITY_FUSION_SIGNAL) + bcg_invalid_signals.update( + signal for signal in BCG_INVALID_SIGNALS if signal in lower_line + ) except BaseException: if process.poll() is None: process.terminate() @@ -1224,11 +1743,67 @@ def _run_benchmark_once_impl( ) return {"model": model_key, "label": label, "error": True, "elapsed_s": elapsed} + if breakable_cuda_graph and (not bcg_capture_detected or bcg_invalid_signals): + reason = ( + ", ".join(sorted(bcg_invalid_signals)) + if bcg_invalid_signals + else "no '[Diffusion BCG] captured' marker" + ) + print( + " ERROR: BCG evidence is invalid: " + f"{reason}. Do not report this run as BCG performance." + ) + return { + "model": model_key, + "label": label, + "quality": quality, + "breakable_cuda_graph": True, + "bcg_capture_detected": bcg_capture_detected, + "bcg_invalid_signals": sorted(bcg_invalid_signals), + "error": True, + "elapsed_s": elapsed, + } + if returncode != 0: print(f" ERROR: exit code {returncode}") return {"model": model_key, "label": label, "error": True, "elapsed_s": elapsed} - metrics = {"model": model_key, "label": label, "elapsed_s": elapsed, "error": False} + output_artifacts = sorted( + path + for path in output_dir.rglob(f"{output_file_name}*") + if path.is_file() and path.suffix.lower() in GENERATED_OUTPUT_SUFFIXES + ) + missing_artifacts = [] + if not perf_path.is_file(): + missing_artifacts.append("perf dump") + if not output_artifacts: + missing_artifacts.append("generated output") + if missing_artifacts: + print( + " ERROR: command returned zero without required benchmark artifacts: " + + ", ".join(missing_artifacts) + ) + return { + "model": model_key, + "label": label, + "quality": quality, + "breakable_cuda_graph": breakable_cuda_graph, + "missing_artifacts": missing_artifacts, + "error": True, + "elapsed_s": elapsed, + } + + metrics = { + "model": model_key, + "label": label, + "quality": quality, + "breakable_cuda_graph": breakable_cuda_graph, + "bcg_capture_detected": bcg_capture_detected, + "elapsed_s": elapsed, + "output_artifacts": [str(path) for path in output_artifacts], + "output_sha256": [_sha256_file(path) for path in output_artifacts], + "error": False, + } if perf_path.exists(): try: with open(perf_path) as f: @@ -1287,12 +1862,53 @@ def _run_benchmark_once_impl( return metrics +def _validate_quality_bcg_output_hashes(results: list[dict]) -> None: + """Reject BCG rows whose generated artifacts differ from eager.""" + for quality in BENCHMARK_QUALITY_LEVELS: + quality_results = [ + result for result in results if result.get("quality") == quality + ] + eager_results = [ + result + for result in quality_results + if not result.get("breakable_cuda_graph") and not result.get("error") + ] + eager_hashes = [ + tuple(result.get("output_sha256", ())) for result in eager_results + ] + if not eager_hashes or any(not hashes for hashes in eager_hashes): + continue + + reference_hashes = eager_hashes[0] + if any(hashes != reference_hashes for hashes in eager_hashes[1:]): + reason = f"eager {quality} output hashes are unstable" + for result in quality_results: + result["error"] = True + result["output_hash_error"] = reason + print(f" ERROR: {reason}; do not use this matrix as BCG evidence.") + continue + + for result in quality_results: + if not result.get("breakable_cuda_graph") or result.get("error"): + continue + output_hashes = tuple(result.get("output_sha256", ())) + if not output_hashes or output_hashes == reference_hashes: + continue + reason = f"BCG {quality} output hash differs from eager" + result["error"] = True + result["output_hash_error"] = reason + print(f" ERROR: {reason}; do not report this row as BCG performance.") + + def run_benchmark_once( model_key: str, label: str, output_dir: Path, warmup: bool = True, torch_compile: bool = False, + quality: str = "lossless", + breakable_cuda_graph: bool = False, + bcg_text_buckets: list[int] | None = None, model_cache_root: Path | None = None, cleanup_model_cache: bool = False, cleanup_ledger_path: Path | None = None, @@ -1310,6 +1926,9 @@ def run_benchmark_once( output_dir, warmup=warmup, torch_compile=torch_compile, + quality=quality, + breakable_cuda_graph=breakable_cuda_graph, + bcg_text_buckets=bcg_text_buckets, model_cache_dir=cache_dir, ) exit_reason = "error" if result.get("error") else "success" @@ -1338,6 +1957,78 @@ def run_benchmark_once( ) +def run_quality_bcg_matrix( + model_key: str, + label: str, + output_dir: Path, + warmup: bool = True, + bcg_text_buckets: list[int] | None = None, + model_cache_root: Path | None = None, + cleanup_model_cache: bool = False, + cleanup_ledger_path: Path | None = None, +) -> list[dict]: + """Run the quality/BCG applicability matrix on one fixed GPU set. + + A high+BCG cell is intentionally retained as a compatibility check. It is + invalid when request-scoped DiT fusions mount after graph capture. + """ + cache_dir = None + exit_reason = "error" + if model_cache_root is not None: + cache_dir = _prepare_model_cache( + model_cache_root, model_key, f"{label}-quality-bcg-matrix" + ) + + cuda_visible_devices = os.environ.get("CUDA_VISIBLE_DEVICES") + if not cuda_visible_devices: + cuda_visible_devices = ",".join( + str(index) for index in pick_idle_gpus(required_gpus_for_model(model_key)) + ) + + results: list[dict] = [] + try: + for mode_label, quality, breakable_cuda_graph in QUALITY_BCG_ABBA_MATRIX: + result = _run_benchmark_once_impl( + model_key, + f"{label}-{mode_label}", + output_dir, + warmup=warmup, + quality=quality, + breakable_cuda_graph=breakable_cuda_graph, + bcg_text_buckets=(bcg_text_buckets if breakable_cuda_graph else None), + model_cache_dir=cache_dir, + cuda_visible_devices=cuda_visible_devices, + ) + results.append(result) + _validate_quality_bcg_output_hashes(results) + exit_reason = ( + "error" if any(result.get("error") for result in results) else "success" + ) + return results + except KeyboardInterrupt: + exit_reason = "interrupted" + raise + finally: + if cleanup_model_cache and cache_dir is not None: + assert model_cache_root is not None + ledger_path = cleanup_ledger_path or output_dir / "cleanup.jsonl" + record = _cleanup_model_cache( + model_cache_root, + cache_dir, + ledger_path, + model_key, + f"{label}-quality-bcg-matrix", + exit_reason, + ) + before = record["before"] + assert isinstance(before, dict) + print( + " Cleaned isolated model cache after the full matrix: " + f"{before['total_bytes']} bytes, " + f"{before['weight_file_count']} weight files; ledger={ledger_path}" + ) + + def print_results_table(results: list[dict]): """Print a compact table for one or more benchmark runs.""" print() @@ -1347,7 +2038,7 @@ def print_results_table(results: list[dict]): print("=" * RESULTS_TABLE_WIDTH) print( - f"{'Model':<24} {'Nightly':<28} {'Label':<12} {'Denoise(s)':>12} {'E2E(s)':>10} {'Peak Mem(GB)':>14}" + f"{'Model':<24} {'Nightly':<28} {'Label':<31} {'Denoise(s)':>12} {'E2E(s)':>10} {'Peak Mem(GB)':>14}" ) print("-" * RESULTS_TABLE_WIDTH) @@ -1359,7 +2050,7 @@ def print_results_table(results: list[dict]): e2e_text = f"{e2e_s:.2f}" if isinstance(e2e_s, float) else "n/a" mem_text = f"{peak_mem:.1f}" if isinstance(peak_mem, float) else "n/a" print( - f"{result['model']:<24} {model_nightly_case_id(result['model']):<28} {result['label']:<12} {denoise_text:>12} {e2e_text:>10} {mem_text:>14}" + f"{result['model']:<24} {model_nightly_case_id(result['model']):<28} {result['label']:<31} {denoise_text:>12} {e2e_text:>10} {mem_text:>14}" ) print("-" * RESULTS_TABLE_WIDTH) @@ -1407,6 +2098,34 @@ def main(): help="Directory for perf dump JSON files", ) parser.add_argument("--no-warmup", action="store_true", help="Skip warmup") + parser.add_argument( + "--quality", + choices=BENCHMARK_QUALITY_LEVELS, + default="lossless", + help="Request quality for a single run (default: lossless).", + ) + parser.add_argument( + "--breakable-cuda-graph", + action="store_true", + help=( + "Run a BCG comparator. The result is invalid unless capture is " + "observed and no disable/failure/signature-miss marker appears." + ), + ) + parser.add_argument( + "--bcg-text-buckets", + type=int, + nargs="+", + help="Optional positive text buckets for a BCG run or matrix.", + ) + parser.add_argument( + "--quality-bcg-matrix", + action="store_true", + help=( + "Run lossless/high Eager-vs-BCG as two ABBA pairs on one GPU set " + "and one task-owned model cache." + ), + ) compile_group = parser.add_mutually_exclusive_group() compile_group.add_argument( "--torch-compile", @@ -1453,6 +2172,17 @@ def main(): output_dir.mkdir(parents=True, exist_ok=True) warmup = not args.no_warmup torch_compile = args.torch_compile and not args.no_torch_compile + if args.quality_bcg_matrix and torch_compile: + parser.error("--quality-bcg-matrix cannot be combined with --torch-compile") + if args.breakable_cuda_graph and torch_compile: + parser.error("--breakable-cuda-graph cannot be combined with --torch-compile") + if args.bcg_text_buckets and not ( + args.breakable_cuda_graph or args.quality_bcg_matrix + ): + parser.error( + "--bcg-text-buckets requires --breakable-cuda-graph or " + "--quality-bcg-matrix" + ) if args.cleanup_model_cache and not args.model_cache_root: parser.error("--cleanup-model-cache requires --model-cache-root") model_cache_root = ( @@ -1466,18 +2196,35 @@ def main(): results = [] for model_key in models_to_run: - results.append( - run_benchmark_once( - model_key, - args.label, - output_dir, - warmup=warmup, - torch_compile=torch_compile, - model_cache_root=model_cache_root, - cleanup_model_cache=args.cleanup_model_cache, - cleanup_ledger_path=cleanup_ledger_path, + if args.quality_bcg_matrix: + results.extend( + run_quality_bcg_matrix( + model_key, + args.label, + output_dir, + warmup=warmup, + bcg_text_buckets=args.bcg_text_buckets, + model_cache_root=model_cache_root, + cleanup_model_cache=args.cleanup_model_cache, + cleanup_ledger_path=cleanup_ledger_path, + ) + ) + else: + results.append( + run_benchmark_once( + model_key, + args.label, + output_dir, + warmup=warmup, + torch_compile=torch_compile, + quality=args.quality, + breakable_cuda_graph=args.breakable_cuda_graph, + bcg_text_buckets=args.bcg_text_buckets, + model_cache_root=model_cache_root, + cleanup_model_cache=args.cleanup_model_cache, + cleanup_ledger_path=cleanup_ledger_path, + ) ) - ) if results: print_results_table(results) diff --git a/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-benchmark-profile/scripts/diffusion_skill_env.py b/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-benchmark-profile/scripts/diffusion_skill_env.py index cffbdebf7..1bf8e5c55 100644 --- a/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-benchmark-profile/scripts/diffusion_skill_env.py +++ b/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-benchmark-profile/scripts/diffusion_skill_env.py @@ -13,6 +13,14 @@ OUTPUT_DIR_NAMES = { def get_repo_root() -> Path: + # Prefer the checkout that owns this skill. A benchmark helper may be + # loaded directly from a secondary worktree while another SGLang install + # is first on sys.path; importing that install would point assets and + # outputs at the wrong repository. + for parent in Path(__file__).resolve().parents: + if (parent / "python" / "sglang" / "__init__.py").is_file(): + return parent + import sglang return Path(sglang.__file__).resolve().parents[2] diff --git a/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-performance/SKILL.md b/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-performance/SKILL.md index 7f130b88a..3d88bc932 100644 --- a/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-performance/SKILL.md +++ b/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-performance/SKILL.md @@ -12,8 +12,11 @@ Before running any `sglang generate` command below inside the diffusion containe - export `HF_TOKEN` first when the selected model lives in a gated Hugging Face repo such as `black-forest-labs/FLUX.*` - export `FLASHINFER_DISABLE_VERSION_CHECK=1` - when a run downloads weights, use a task-owned cache and delete that model's - cache after its eager/compile/BCG/profile group finishes; the benchmark skill - provides `--model-cache-root --cleanup-model-cache` plus a cleanup ledger + cache after its eager/BCG/quality/profile group finishes; the benchmark + skill's `--quality-bcg-matrix --model-cache-root --cleanup-model-cache` + keeps one cache for the group and writes a zero-residual cleanup ledger +- hold one idle GPU set for the complete A/B matrix and verify no foreign + process appears at run boundaries - `cd` to the repo root resolved from `sglang.__file__` ## Native Backend Gate @@ -37,7 +40,7 @@ These options are intended to preserve output quality. In practice, some paths ( |---|---|---|---|---| | **Performance Mode** | `--performance-mode auto\|speed\|memory\|manual` (`--mode` alias) | Applies model-aware residency, FSDP/CFG, and compile defaults without overriding explicit flags. `auto` is the safe default; `speed` favors GPU residency; `memory` favors offload; `manual` leaves performance args explicit. | Fastest way to establish a sensible deployment baseline | `speed` may OOM and enables `torch.compile` only when the model deployment config allows it. Explicit offload/FSDP/parallelism/compile flags win. Use `manual` for controlled A/B benchmarks. | | **torch.compile** | `--enable-torch-compile` | Applies `torch.compile` to the DiT forward pass. Treat it as a measured comparator, not an assumed upgrade. | Model- and shape-dependent; recent B300 coverage found eager or valid BCG faster or within 1% for every valid compile control | First request is slow and some models time out or drift numerically. Keep eager as the ground truth, use a warmup watchdog, and validate the target model. See the [H200/B300 survey](https://github.com/BBuf/how-to-optim-algorithm-in-cuda/issues/21). | -| **Breakable CUDA Graph** | `--enable-breakable-cuda-graph` plus optional `--warmup-resolutions ` and `--bcg-text-buckets ...` | Captures fixed-resolution DiT segments while leaving attention/collectives eager, reducing launch overhead on supported pipelines. | Large on launch-bound paths; merged SANA and LTX-2 cases show material e2e gains | Mutually exclusive with `torch.compile` and Cache-DiT; BCG takes priority. The model's default resolution is captured automatically; declare every additional production resolution. Current support is model-specific (Ideogram4, LTX-2/2.3, LongCat-Image, MiniMax-H3, Qwen-Image, SANA1.5, SANA-Video, Z-Image, GLM-Image); benchmark before keeping it. | +| **Breakable CUDA Graph** | `--enable-breakable-cuda-graph` plus optional `--warmup-resolutions ` and `--bcg-text-buckets ...` | Captures fixed-resolution DiT segments while leaving attention/collectives eager, reducing launch overhead on supported pipelines. | Large on launch-bound paths; merged SANA and LTX-2 cases show material e2e gains | Mutually exclusive with `torch.compile` and Cache-DiT; BCG takes priority. The model's default resolution is captured automatically; declare every additional production resolution. Current support is model-specific (Ideogram4, LTX-2/2.3, LongCat-Image, MiniMax-H3, Qwen-Image, SANA1.5, SANA-Video, Z-Image, GLM-Image), but an allowlisted model is not automatically a validated recipe. A valid run must log capture and no disable/failure/signature miss. `--warmup-resolutions` covers only `WxH`; video frame/conditioning mismatches can still fall back to Eager. MiniMax-H3 remains eager in the validated deployment because prompt-dependent packed host boundaries can miss the captured signature. | | **Warmup** | `--warmup-mode request` | Runs dummy forward passes to warm up CUDA caches, JIT, and `torch.compile`. Eliminates cold-start penalty. | Removes first-request latency spike | Adds startup time. Without `--warmup-resolutions`, warmup happens on first request. | | **Warmup Resolutions** | `--warmup-resolutions 256x256 720x720` | Pre-compiles and warms up specific resolutions at server startup (instead of lazily on first request). | Faster first request per resolution | Each resolution adds to startup time. Serving mode only; useful when you know your target resolutions in advance. | | **Multi-GPU (SP)** | `--num-gpus N --ulysses-degree N` | Sequence parallelism across GPUs. Shards sequence tokens (not frames) to minimize padding. | Near-linear scaling with N GPUs | Requires NCCL; inter-GPU bandwidth matters. `ulysses_degree * ring_degree = sp_degree`. For Wan2.2 video, start by benchmarking pure Ulysses before assuming a mixed Ulysses/Ring layout is fastest. | @@ -59,7 +62,7 @@ These options **trade output quality** for speed or VRAM savings. Results will d | Option | CLI Flag / Env Var | What It Does | Speedup | Quality Impact / Limitations | |---|---|---|---|---| -| **Request Quality Fast Paths** | `--quality high` (`lossless` is default) | Mounts model-owned accelerated DiT/VAE paths that are validated for high quality but are not bit-exact to the reference path. | Model- and shape-specific | Support is per model and may be a no-op. Keep `--quality lossless` as the A/B ground truth. Do not confuse this with `--output-quality`, which controls file compression. | +| **Request Quality Fast Paths** | `--quality high` (`lossless` is default) | Mounts model-owned accelerated DiT/VAE paths that are validated for high quality but are not bit-exact to the reference path. | Model- and shape-specific | Support is per model and may be a no-op. Keep `--quality lossless` as the A/B ground truth. Report aggregate and worst-frame SSIM/PSNR; defaults are 0.95/28 dB for images and 0.92/24 dB for video unless checked-in model metadata overrides them. Do not confuse this with `--output-quality`, which controls file compression. | | **Approximate Attention** | Server-wide: `--attention-backend sage_attn` / `sage_attn_3` / `sliding_tile_attn` / `video_sparse_attn` / `sparse_video_gen_2_attn` / `vmoba_attn` / `sla_attn` / `sage_sla_attn`. Per-request (dense drop-ins only): `--attention-backend-override sage_attn` sampling param / API `extra_body` — valid values `fa`, `torch_sdpa`, `sage_attn`, `sage_attn_3`; rejected (with a log) under BCG, torch.compile, sparse server backends, or a non-ring-capable target with ring parallelism. | Replaces exact attention with approximate or sparse variants. `sage_attn`: INT8/FP8 quantized Q·K; `sliding_tile_attn`: spatial-temporal tile skipping; others: model-specific sparse patterns. | ~1.5–2x on attention (varies by backend) | Quality degradation varies by backend and model. `sage_attn` is the most general; sparse backends (`sliding_tile_attn`, `video_sparse_attn`, etc.) are video-model-specific, may require config files (e.g. `--mask-strategy-file-path` for STA), and are server-level only. Requires corresponding packages installed. | | **Cache-DiT** | Native: per-request `--enable-cache-dit true\|false` + `--cache-dit-params ` (sampling params; also via API `extra_body`). `SGLANG_CACHE_DIT_ENABLED` / `SGLANG_CACHE_DIT_*` env vars are the server-wide defaults for requests that leave them unset. Diffusers backend: `--backend diffusers --cache-dit-config ` | Caches intermediate residuals across denoising steps and skips redundant computations via DBCache, TaylorSeer, and optional SCM. | ~1.5-2x on supported models | Quality depends on cache policy. Incompatible with `--dit-layerwise-offload`. Do not pass `--cache-dit-config` for native SGLang tuning unless you are intentionally using the diffusers backend flow. | | **CFG Gating** | Per-request `--cfg-gate-step 0.5` (sampling param; also via API `extra_body`). `SGLANG_DIFFUSION_CFG_GATE_STEP` is the server-wide default (1.0 = off). | After the given fraction of denoising steps, reuses the cached cond-uncond residual instead of running the unconditional branch each step. | Up to ~2x on the gated tail of CFG models (skips one of two branches) | Lossy; no-op without classifier-free guidance or with `--enable-cfg-parallel`. Lower fractions gate earlier and drift more. | @@ -243,6 +246,21 @@ sglang serve --model-path Efficient-Large-Model/SANA1.5_1.6B_1024px_diffusers \ Keep `torch.compile` off, declare every production resolution, and benchmark the exact prompt-length distribution. Add `--bcg-text-buckets` only when the default buckets create excessive padding or miss a served prompt signature. +Do not keep the timing unless the log contains `[Diffusion BCG] captured` and +contains no disable, capture-failure, or `serving signature MISSED` message. +For video, also match the captured frame and conditioning shape; `WxH` alone +does not prove replay. + +For a repeated discovery sweep, use the benchmark/profile helper. This runs +lossless and high-quality Eager/BCG ABBA pairs on one GPU set, then deletes the +model group cache once: + +```bash +python3 python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-benchmark-profile/scripts/bench_diffusion_denoise.py \ + --model --quality-bcg-matrix \ + --model-cache-root /path/to/task-owned/model-caches \ + --cleanup-model-cache +``` ### Compare request-scoped high-quality fast paths @@ -357,7 +375,7 @@ Use these as first commands to benchmark, not as universal winners. | Model family | First performance shape | Starting flags | Notes | |---|---|---|---| -| MiniMax-H3 | 1344x768 resolved canvas, 5 seconds / 124 frames at 24 fps, 50 joint video/audio steps | H200: `--num-gpus 4 --ulysses-degree 4 --performance-mode speed --enable-torch-compile false`; H100: TP2 + Ulysses2 | Root ID plus `--model-variant fl2va` for T2VA/FL2VA or `ref2va` for Ref2VA. Ulysses only; no Ring/CFG/SageAttention. Preserve tiled video-VAE decode. Profile joint denoise, video VAE, audio VAE/vocoder, encoder, and collectives separately. | +| MiniMax-H3 | 1344x768 resolved canvas, 5 seconds / 124 frames at 24 fps, 50 joint video/audio steps | H200: `--num-gpus 4 --ulysses-degree 4 --performance-mode speed --enable-torch-compile false --enable-breakable-cuda-graph false`; H100: TP2 + Ulysses2 | Root ID plus `--model-variant fl2va` for T2VA/FL2VA or `ref2va` for Ref2VA. Ulysses only; no Ring/CFG/SageAttention. Preserve tiled video-VAE decode. BCG is not part of the validated H3 recipe: warmup and serving can have different packed host boundaries, and a replay-capable experiment must still beat eager without excessive graph memory. Profile joint denoise, video VAE, audio VAE/vocoder, encoder, and collectives separately. | | FLUX.1 / FLUX.2 image | 1024x1024, runtime-default steps/guidance, 1 GPU | `--enable-torch-compile --warmup-mode request --dit-layerwise-offload false` | `black-forest-labs/FLUX.*` repos are gated; for FP8/NVFP4 use validated `--transformer-path` or `--transformer-weights-path` flows from the quant skill. | | FLUX.2 Klein / Klein Base | 1024x1024, runtime-default steps/guidance, 1 GPU | `--enable-torch-compile --warmup-mode request --dit-layerwise-offload false` | Current registry has `black-forest-labs/FLUX.2-klein-4B`, `FLUX.2-klein-9B`, and base variants. Klein is step-distilled; Klein Base is not. | | Qwen-Image / Qwen-Image-Edit | 1024x1024, runtime-default steps/guidance, 1 GPU | `--enable-torch-compile --warmup-mode request`; optionally native `SGLANG_CACHE_DIT_ENABLED=true` | Cache-DiT is lossy. For edit tasks, keep reference image, seed, and output size fixed. | @@ -366,7 +384,7 @@ Use these as first commands to benchmark, not as universal winners. | Wan2.2 A14B T2V/I2V | 1280x720, 81 frames | Nightly: `--num-gpus 4 --enable-cfg-parallel --ulysses-degree 2 --text-encoder-cpu-offload --pin-cpu-memory` | For lowest latency, also benchmark pure Ulysses on the same GPUs. | | Wan2.2 TI2V 5B | 1280x720, 81 frames, 1 GPU | `--enable-torch-compile --warmup-mode request` | Keep the input image and motion prompt fixed when comparing sparse attention or Cache-DiT. | | Wan2.1 / FastWan / TurboWan variants | 480p or 720p video, family defaults | `--enable-torch-compile --warmup-mode request`; add `--ulysses-degree` / CFG parallel only after measuring | Current registry includes Wan2.1, FastWan2.1, FastWan2.2 TI2V, TurboWan2.1, TurboWan2.2 I2V, and Wan2.1-Fun InP. Use the compatibility matrix and benchmark presets before choosing topology. | -| Cosmos3 Nano / Super | T2I: 1024x1024 with `--num-frames 1`; T2V/I2V: 480p/720p video | `SGLANG_DISABLE_COSMOS3_GUARDRAILS=1` for benchmark isolation; `--enable-torch-compile --warmup-mode request` | One checkpoint serves T2I/T2V/I2V. Mode is request-driven: `num_frames == 1` means T2I, `--image-path` means I2V. | +| Cosmos3 Nano / Super | T2I: 1024x1024 with `--num-frames 1`; T2V/I2V: 480p/720p video | Start with `--performance-mode auto --warmup-mode request`; use `SGLANG_DISABLE_COSMOS3_GUARDRAILS=1` only for benchmark isolation, and compare compile separately | One checkpoint serves T2I/T2V/I2V. Mode is request-driven: `num_frames == 1` means T2I, `--image-path` means I2V. On GPUs with at least 120 GiB available, auto mode keeps Cosmos3 Nano's DiT and VAE resident; a 1xH200 832x480x9f, 4-step eager ABBA reduced e2e from 1.576 to 0.428 seconds with exact output parity. The override is Nano-only; keep Super on its conservative multi-GPU policy. | | Cosmos3 Edge / distilled Super | Edge T2I: 640x640, 35 steps, 1 GPU; distilled Super T2I: 640x640, fixed 4-step schedule, 4 GPUs | Start eager with `--performance-mode manual`; use `SGLANG_DISABLE_COSMOS3_GUARDRAILS=1` only for benchmark isolation | Edge is trained for 256p/480p shapes. Distilled checkpoints own their sigma schedule and force guidance 1.0; do not override steps or flow shift. Do not retry the closed experimental Cosmos BCG path without a new lifecycle design. | | Ideogram 4 FP8/NVFP4 | 1024x1024, native preset defaults | `--enable-torch-compile --warmup-mode request` | Do not set `--num-inference-steps` or `--guidance-scale` directly unless you also update the Ideogram preset; sampling params derive them from `preset`. | | ERNIE-Image / GLM-Image / SANA / SD3 | 1024-class image, family defaults | `--enable-torch-compile --warmup-mode request`; disable offload only after checking VRAM | Treat these as current native image families. Start with benchmark/profile presets for ERNIE, GLM, and SANA; use registry/config defaults for SD3 unless you add a new preset. | @@ -378,7 +396,7 @@ Use these as first commands to benchmark, not as universal winners. | JoyAI-Image-Edit | 1024-class TI2I, 40 steps, guidance 4.0 | `--backend=sglang --num-gpus 2 --enable-cfg-parallel --ulysses-degree 1 --enable-torch-compile --warmup-mode request --dit-layerwise-offload false --dit-cpu-offload false` | Newly supported image-edit path. Keep the input image, prompt, seed, and output size fixed; 2-GPU CFG parallel is the validated H100 starting point. | | FireRed-Image-Edit 1.0 / 1.1 | 1024x1024 image edit, 40 steps, guidance 4.0 | `--backend=sglang --num-gpus 2 --enable-cfg-parallel --ulysses-degree 1 --enable-torch-compile --warmup-mode request --dit-layerwise-offload false --dit-cpu-offload false` | Uses the native `QwenImageEditPlusPipeline` path. 2-GPU CFG parallel is the validated H100 starting point; benchmark 1.0 and 1.1 separately because checkpoint differences can change denoise latency. | | Hunyuan3D-2 shape | Shape generation, 50 steps, guidance 5.0 | `--backend=sglang --enable-torch-compile --warmup-mode request --dit-layerwise-offload false --dit-cpu-offload false` | Focus on `Hunyuan3DShapeDenoisingStage`; keep mesh export/paint timings separate from denoise. | -| LingBot Video MoE 30B | 384x640, 17 frames, 12 steps for the current GPU case | `--model-path robbyant/lingbot-video-moe-30b-a3b --text-encoder-cpu-offload` | Native T2V path. Prompts are structured JSON captions, not raw free text; keep that contract when comparing latency or quality. | +| LingBot Video MoE 30B | 384x640, 17 frames, 12 steps for the current GPU case | `--model-path robbyant/lingbot-video-moe-30b-a3b --text-encoder-cpu-offload` | Native T2V path. Prompts are structured JSON captions, not raw free text; keep that contract when comparing latency or quality. Main still expands RMSNorm into PyTorch reduction chains; #35969 is an open `quality=high` Triton-dispatch candidate, not a current-main option until merged. | | MOVA / Helios / LingBot World | Use the benchmark/profile presets or server test cases first | `--enable-torch-compile --warmup-mode request`; pin offload and topology flags explicitly | These video/realtime families have model-specific stages and condition handling. For LingBot World causal serving, keep `--kv-cache-quant off` as the exact cache baseline before testing INT4/INT2. | ## Historical PR Watchlist @@ -392,13 +410,19 @@ about whether the work has merged: default-resolution BCG warmup, #34210 Z-Image BCG correctness, #34305/#34314 Ideogram eager fusions, #34584 Wan TI2V modulation/RoPE, #34616 FLUX2, #34617 Hunyuan, #34619 GLM, #34620 ERNIE, #34928 SANA, #34929 LTX2.3, - and #34932 Cosmos3. Re-check open/merged state before reusing a path. + #34932 Cosmos3, #35724 LongCat BCG, #35728 SANA-Video high-quality linear + attention, and #35729 SANA-Video BCG. #35961/#35969/#35981 are open + SANA-Video, LingBot, and Wan VAE candidates. Re-check open/merged state + before reusing a path. - VAE/decode: #22531 LTX2 parallel VAE, #20927 batched tiled VAE decode. - Runtime/parallel/cache: #22805 FLUX.2 packed QKV for A2A, #21742 hybrid attention schedule, #24053 USP replicated-prefix fix, #21613 TeaCache refactor, #24227 WanVideo TeaCache fix, #18764 dynamic batching, #24200 disaggregated diffusion. ## Tips - **Benchmarking**: establish eager first (`--performance-mode manual`, compile/BCG/cache off), always use `--warmup-mode request`, and look for the line ending with `(with warmup excluded)` for accurate timing. Add compile or BCG as separate labeled controls. +- **PR gate**: use repeated same-GPU ABBA measurements and saved-request wall + time. Require at least 1.5% mean e2e improvement for this optimization sweep; + attach a representative baseline/candidate profile and generated-media A/B. - **Checkpoint cleanup**: finish every variant for one model, then delete only its task-owned cache and verify the cleanup ledger reports zero residual weight files. Never point cleanup at a shared Hugging Face or ModelScope cache. diff --git a/python/sglang/multimodal_gen/test/unit/test_diffusion_benchmark_skill.py b/python/sglang/multimodal_gen/test/unit/test_diffusion_benchmark_skill.py index bac804d9c..0190b7102 100644 --- a/python/sglang/multimodal_gen/test/unit/test_diffusion_benchmark_skill.py +++ b/python/sglang/multimodal_gen/test/unit/test_diffusion_benchmark_skill.py @@ -37,7 +37,35 @@ def _load_benchmark_module(temp_root: Path): return module +def _load_skill_env_module(): + script_path = ( + Path(__file__).resolve().parents[2] + / ".claude" + / "skills" + / "sglang-diffusion-benchmark-profile" + / "scripts" + / "diffusion_skill_env.py" + ) + spec = importlib.util.spec_from_file_location( + "test_diffusion_skill_env", script_path + ) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + class TestDiffusionBenchmarkSkill(unittest.TestCase): + def test_skill_env_prefers_own_worktree_over_installed_package(self): + module = _load_skill_env_module() + installed = types.ModuleType("sglang") + installed.__file__ = "/sgl-workspace/sglang/python/sglang/__init__.py" + + with patch.dict(sys.modules, {"sglang": installed}): + repo_root = module.get_repo_root() + + self.assertEqual(repo_root, Path(__file__).resolve().parents[5]) + def test_nightly_presets_remain_aligned(self): with tempfile.TemporaryDirectory() as tmpdir: module = _load_benchmark_module(Path(tmpdir)) @@ -60,8 +88,27 @@ class TestDiffusionBenchmarkSkill(unittest.TestCase): expected = { "longcat-image", "sana-video", + "sana-wm-bidirectional", + "sana-wm-streaming", "lingbot-video-moe", + "lingbot-world", + "lingbot-world-v2", + "fastwan21-t2v-1.3b", + "wan22-t2v-nvfp4", + "krea2-turbo", + "krea2-raw", + "ideogram4-fast", + "ideogram4-instant", + "longlive2-t2v", + "longlive2-i2v", + "fast-hunyuan", + "turbowan21-t2v-1.3b", + "helios-mid", + "helios-distilled", + "joy-echo", "cosmos3-edge-t2i", + "cosmos3-super-t2v-cfg2tp2", + "cosmos3-super-i2v", "cosmos3-super-t2i-distilled", "ltx25", "ltx25-diffusion-decoder", @@ -71,6 +118,7 @@ class TestDiffusionBenchmarkSkill(unittest.TestCase): eager_cmd = module.build_sglang_cmd("longcat-image") self.assertNotIn("--enable-torch-compile", eager_cmd) self.assertIn("--enable-prompt-rewrite=false", eager_cmd) + self.assertIn("--quality=lossless", eager_cmd) compiled_cmd = module.build_sglang_cmd("longcat-image", torch_compile=True) self.assertIn("--enable-torch-compile", compiled_cmd) @@ -78,6 +126,130 @@ class TestDiffusionBenchmarkSkill(unittest.TestCase): h3_cmd = module.build_sglang_cmd("minimax-h3-t2va", torch_compile=True) self.assertNotIn("--enable-torch-compile", h3_cmd) + fastwan_cmd = module.build_sglang_cmd("fastwan21-t2v-1.3b") + self.assertIn("--num-frames=61", fastwan_cmd) + self.assertIn("--num-inference-steps=3", fastwan_cmd) + self.assertIn("--dit-layerwise-offload=false", fastwan_cmd) + + wan_nvfp4_cmd = module.build_sglang_cmd("wan22-t2v-nvfp4") + self.assertIn( + "--model-path=nvidia/Wan2.2-T2V-A14B-Diffusers-NVFP4", + wan_nvfp4_cmd, + ) + self.assertIn("--num-frames=81", wan_nvfp4_cmd) + self.assertIn("--dit-layerwise-offload=false", wan_nvfp4_cmd) + self.assertEqual(module.required_gpus_for_model("wan22-t2v-nvfp4"), 1) + + krea_raw_cmd = module.build_sglang_cmd("krea2-raw") + self.assertIn("--num-inference-steps=50", krea_raw_cmd) + self.assertIn("--guidance-scale=4.5", krea_raw_cmd) + + cosmos_i2v_cmd = module.build_sglang_cmd("cosmos3-super-i2v") + self.assertIn( + "--model-path=nvidia/Cosmos3-Super-Image2Video", cosmos_i2v_cmd + ) + self.assertIn("--num-gpus=2", cosmos_i2v_cmd) + self.assertIn("--tp-size=2", cosmos_i2v_cmd) + self.assertIn("--num-frames=81", cosmos_i2v_cmd) + + cosmos_cfg_cmd = module.build_sglang_cmd("cosmos3-super-t2v-cfg2tp2") + self.assertIn("--model-path=nvidia/Cosmos3-Super", cosmos_cfg_cmd) + self.assertIn("--num-gpus=4", cosmos_cfg_cmd) + self.assertIn("--tp-size=2", cosmos_cfg_cmd) + + sana_wm_dense_cmd = module.build_sglang_cmd("sana-wm-bidirectional") + self.assertIn( + "--model-path=Efficient-Large-Model/SANA-WM_bidirectional", + sana_wm_dense_cmd, + ) + self.assertIn("--num-inference-steps=20", sana_wm_dense_cmd) + self.assertNotIn("--streaming", sana_wm_dense_cmd) + + sana_wm_streaming_cmd = module.build_sglang_cmd("sana-wm-streaming") + self.assertIn("--streaming", sana_wm_streaming_cmd) + self.assertIn("--refiner-chunked", sana_wm_streaming_cmd) + self.assertIn("--action=w-16,wl-16,l-16", sana_wm_streaming_cmd) + + lingbot_world_cmd = module.build_sglang_cmd("lingbot-world") + self.assertIn( + "--model-path=robbyant/lingbot-world-fast-diffusers", + lingbot_world_cmd, + ) + self.assertIn("--num-frames=9", lingbot_world_cmd) + self.assertIn("--warmup-mode=off", lingbot_world_cmd) + self.assertIn("--config=", " ".join(lingbot_world_cmd)) + self.assertEqual( + module.MODELS["lingbot-world"]["config_overrides"]["actions"], + [["w"] for _ in range(9)], + ) + + lingbot_world_v2_cmd = module.build_sglang_cmd("lingbot-world-v2") + self.assertIn( + "--model-path=robbyant/lingbot-world-v2-14b-causal-fast-diffusers", + lingbot_world_v2_cmd, + ) + self.assertIn("--num-frames=9", lingbot_world_v2_cmd) + self.assertIn("--num-inference-steps=4", lingbot_world_v2_cmd) + + ideogram_cmd = module.build_sglang_cmd("ideogram4-instant") + self.assertFalse( + any(arg.startswith("--num-inference-steps") for arg in ideogram_cmd) + ) + + longlive_i2v_cmd = module.build_sglang_cmd("longlive2-i2v") + self.assertIn("--num-frames=61", longlive_i2v_cmd) + self.assertTrue( + any(arg.startswith("--image-path=") for arg in longlive_i2v_cmd) + ) + + joy_echo_cmd = module.build_sglang_cmd("joy-echo") + self.assertIn("--num-gpus=2", joy_echo_cmd) + self.assertIn("--ulysses-degree=2", joy_echo_cmd) + config_arg = next( + arg for arg in joy_echo_cmd if arg.startswith("--config=") + ) + config = json.loads(Path(config_arg.removeprefix("--config=")).read_text()) + self.assertFalse(config["enable_memory_bank"]) + + def test_quality_and_bcg_comparators_are_explicit_and_exclusive(self): + with tempfile.TemporaryDirectory() as tmpdir: + module = _load_benchmark_module(Path(tmpdir)) + + high_cmd = module.build_sglang_cmd("longcat-image", quality="high") + self.assertIn("--quality=high", high_cmd) + self.assertNotIn("--enable-breakable-cuda-graph", high_cmd) + + bcg_cmd = module.build_sglang_cmd( + "longcat-image", + breakable_cuda_graph=True, + bcg_text_buckets=[256, 512], + ) + self.assertIn("--enable-breakable-cuda-graph", bcg_cmd) + self.assertEqual( + bcg_cmd[bcg_cmd.index("--warmup-resolutions") + 1], "1024x1024" + ) + bucket_index = bcg_cmd.index("--bcg-text-buckets") + self.assertEqual( + bcg_cmd[bucket_index + 1 : bucket_index + 3], ["256", "512"] + ) + + for _, quality, breakable_cuda_graph in module.QUALITY_BCG_ABBA_MATRIX: + module.build_sglang_cmd( + "longcat-image", + quality=quality, + breakable_cuda_graph=breakable_cuda_graph, + bcg_text_buckets=[256, 512] if breakable_cuda_graph else None, + ) + + with self.assertRaisesRegex(ValueError, "comparators"): + module.build_sglang_cmd( + "longcat-image", + torch_compile=True, + breakable_cuda_graph=True, + ) + with self.assertRaisesRegex(ValueError, "requires"): + module.build_sglang_cmd("longcat-image", bcg_text_buckets=[256]) + def test_isolated_cache_cleanup_writes_zero_residual_ledger(self): with tempfile.TemporaryDirectory() as tmpdir: temp_root = Path(tmpdir) @@ -180,6 +352,140 @@ class TestDiffusionBenchmarkSkill(unittest.TestCase): ) self.assertEqual(ledger["exit_reason"], "error") + def test_zero_exit_without_artifacts_is_invalid(self): + with tempfile.TemporaryDirectory() as tmpdir: + temp_root = Path(tmpdir) + module = _load_benchmark_module(temp_root) + output_dir = temp_root / "outputs" + output_dir.mkdir() + + with patch.object(module.subprocess, "Popen") as popen: + popen.return_value.stdout = iter(()) + popen.return_value.wait.return_value = 0 + result = module._run_benchmark_once_impl( + "sana-video", + "missing-artifacts", + output_dir, + warmup=False, + cuda_visible_devices="0", + ) + + command = popen.call_args.args[0] + self.assertIn("--output-path", command) + self.assertIn("--output-file-name", command) + self.assertTrue(result["error"]) + self.assertEqual( + result["missing_artifacts"], ["perf dump", "generated output"] + ) + + def test_high_bcg_rejects_quality_fusion_mounted_after_capture(self): + with tempfile.TemporaryDirectory() as tmpdir: + temp_root = Path(tmpdir) + module = _load_benchmark_module(temp_root) + output_dir = temp_root / "outputs" + output_dir.mkdir() + + with patch.object(module.subprocess, "Popen") as popen: + popen.return_value.stdout = iter( + ( + "[Diffusion BCG] captured 3 segment(s)\n", + "Mounted LTX-2 fused RMSNorm+modulate for quality=high\n", + ) + ) + popen.return_value.wait.return_value = 0 + result = module._run_benchmark_once_impl( + "longcat-image", + "bcg-high", + output_dir, + warmup=False, + quality="high", + breakable_cuda_graph=True, + cuda_visible_devices="0", + ) + + self.assertTrue(result["error"]) + self.assertEqual( + result["bcg_invalid_signals"], + [module.BCG_LATE_QUALITY_FUSION_SIGNAL], + ) + + def test_quality_bcg_matrix_reuses_one_gpu_set_and_cleans_once(self): + with tempfile.TemporaryDirectory() as tmpdir: + temp_root = Path(tmpdir) + module = _load_benchmark_module(temp_root) + cache_root = temp_root / "model-caches" + output_dir = temp_root / "outputs" + output_dir.mkdir() + calls = [] + + def fake_run(model_key, label, _output_dir, **kwargs): + calls.append((model_key, label, kwargs)) + cache_dir = kwargs["model_cache_dir"] + weight_path = cache_dir / "hub" / "model.safetensors" + weight_path.parent.mkdir(parents=True, exist_ok=True) + weight_path.write_bytes(b"weights") + return {"model": model_key, "label": label, "error": False} + + with patch.object(module, "_run_benchmark_once_impl", side_effect=fake_run): + results = module.run_quality_bcg_matrix( + "sana-video", + "h200", + output_dir, + model_cache_root=cache_root, + cleanup_model_cache=True, + ) + + self.assertEqual(len(results), 8) + self.assertEqual( + [ + (call[2]["quality"], call[2]["breakable_cuda_graph"]) + for call in calls + ], + [ + (quality, breakable_cuda_graph) + for _, quality, breakable_cuda_graph in module.QUALITY_BCG_ABBA_MATRIX + ], + ) + self.assertEqual({call[2]["cuda_visible_devices"] for call in calls}, {"0"}) + self.assertEqual( + {call[2]["model_cache_dir"] for call in calls}, + {calls[0][2]["model_cache_dir"]}, + ) + self.assertFalse(calls[0][2]["model_cache_dir"].exists()) + ledger = json.loads( + (output_dir / "cleanup.jsonl").read_text(encoding="utf-8") + ) + self.assertEqual(ledger["exit_reason"], "success") + self.assertEqual(ledger["before"]["weight_file_count"], 1) + self.assertEqual(ledger["after"]["weight_file_count"], 0) + + def test_quality_bcg_matrix_rejects_output_hash_mismatch(self): + with tempfile.TemporaryDirectory() as tmpdir: + module = _load_benchmark_module(Path(tmpdir)) + results = [ + { + "quality": "lossless", + "breakable_cuda_graph": False, + "output_sha256": ["eager"], + "error": False, + }, + { + "quality": "lossless", + "breakable_cuda_graph": True, + "output_sha256": ["bcg"], + "error": False, + }, + ] + + module._validate_quality_bcg_output_hashes(results) + + self.assertFalse(results[0]["error"]) + self.assertTrue(results[1]["error"]) + self.assertEqual( + results[1]["output_hash_error"], + "BCG lossless output hash differs from eager", + ) + if __name__ == "__main__": unittest.main()