diff --git a/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-add-model/SKILL.md b/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-add-model/SKILL.md index 284021205..4f0bb45c1 100644 --- a/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-add-model/SKILL.md +++ b/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-add-model/SKILL.md @@ -120,7 +120,7 @@ Once you have the reference code, study it thoroughly: **Before creating any new files, check whether an existing pipeline or stage can be reused or extended.** Only create new pipelines/stages when the existing ones would require extensive modifications or when no similar implementation exists. Specifically: -1. **Compare the new model's architecture against existing pipelines** before creating files. Current native families include MiniMax-H3, Krea-2, LTX-2/2.3, HunyuanVideo/FastHunyuan, Wan/FastWan/TurboWan/LingBot World/LingBot Video MoE, MOVA, FLUX/FLUX.2/Klein, Z-Image, Qwen-Image/edit/layered, GLM-Image, SD3, Hunyuan3D, Helios, Cosmos3, SANA/SANA-WM, FireRed, ERNIE-Image, JoyAI, and Ideogram4. If the new model shares most of its structure with an existing one (e.g., same text encoders, similar latent format, compatible denoising loop), prefer: +1. **Compare the new model's architecture against existing pipelines** before creating files. Current native families include MiniMax-H3, Krea-2, LTX-2/2.3/2.5, HunyuanVideo/FastHunyuan, Wan/FastWan/TurboWan/LingBot World/LingBot Video MoE, MOVA, FLUX/FLUX.2/Klein, LongCat-Image, Z-Image, Qwen-Image/edit/layered, GLM-Image, SD3, Hunyuan3D, Helios, Cosmos3 Nano/Super/Edge/distilled, SANA/SANA-Video/SANA-WM, FireRed, ERNIE-Image, JoyAI, and Ideogram4. If the new model shares most of its structure with an existing one (e.g., same text encoders, similar latent format, compatible denoising loop), prefer: - Adding a new config variant to the existing pipeline rather than creating a new pipeline class - Reusing the existing `BeforeDenoisingStage` with minor parameter differences - Using `add_standard_t2i_stages()` / `add_standard_ti2i_stages()` / `add_standard_ti2v_stages()` if the model fits standard patterns @@ -583,6 +583,7 @@ After implementation, **you must verify that the generated output is not noise** | GLM-Image | `runtime/pipelines/glm_image.py` | `stages/model_specific_stages/glm_image.py` | `configs/pipeline_configs/glm_image.py` | | Qwen-Image-Layered | `runtime/pipelines/qwen_image.py` (`QwenImageLayeredPipeline`) | `stages/model_specific_stages/qwen_image_layered.py` | `configs/pipeline_configs/qwen_image.py` (`QwenImageLayeredPipelineConfig`) | | Cosmos3 | `runtime/pipelines/cosmos3_pipeline.py` | `stages/model_specific_stages/cosmos3.py` | `configs/pipeline_configs/cosmos3.py` | +| LongCat-Image | `runtime/pipelines/longcat_image.py` | `stages/model_specific_stages/longcat_image.py` | `configs/pipeline_configs/longcat_image.py` | | ErnieImage | `runtime/pipelines/ernie_image.py` | `stages/model_specific_stages/ernie_image_pe.py` | `configs/pipeline_configs/ernie_image.py` | | Hunyuan3D | `runtime/pipelines/hunyuan3d_pipeline.py` | `stages/model_specific_stages/hunyuan3d/` | `configs/pipeline_configs/hunyuan3d.py` | | SANA-WM | `runtime/pipelines/sana_wm_pipeline.py`, `sana_wm_realtime_pipeline.py` | `stages/model_specific_stages/sana_wm/` | `configs/pipeline_configs/sana_wm.py` | @@ -600,8 +601,9 @@ After implementation, **you must verify that the generated output is not noise** | Z-Image | `runtime/pipelines/zimage_pipeline.py` | Uses standard image pipeline stages plus Z-Image-specific config/model code | | Ideogram4 | `runtime/pipelines/ideogram.py` | Uses dedicated text encoding and denoising stages while keeping standard latent prep | | SANA | `runtime/pipelines/sana.py` | Spatial image pipeline; reuse the spatial image config pattern | +| SANA-Video | `runtime/pipelines/sana_video.py` | Native 3D transformer with model-specific text encoding and otherwise standard T2V stages | | Stable Diffusion 3/3.5 | `runtime/pipelines/stable_diffusion_3.py` | Spatial image pipeline; compare scheduler, VAE scale, and conditioning layout | -| LTX-2 / LTX-2.3 | `runtime/pipelines/ltx_2_pipeline.py` | Video pipeline family with one-stage, two-stage, and HQ variants | +| LTX-2 / LTX-2.3 / LTX-2.5 | `runtime/pipelines/ltx_2_pipeline.py` | Video pipeline family with one-stage, two-stage, HQ, joint audio/video, and optional LTX-2.5 diffusion-decoder variants; prefer config/loader specialization over a new pipeline | | Helios | `runtime/pipelines/helios_pipeline.py` | Video pipeline family with custom denoising and decoding stages | | FireRed/JoyAI image edit | `runtime/pipelines/qwen_image.py`, `runtime/pipelines/joy_image.py` | FireRed reuses Qwen edit-plus config; JoyAI has its own edit pipeline | | Wan | `runtime/pipelines/wan_pipeline.py` | Uses `add_standard_ti2v_stages()` | diff --git a/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-benchmark-profile/SKILL.md b/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-benchmark-profile/SKILL.md index ee9459409..6a16fd4f6 100644 --- a/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-benchmark-profile/SKILL.md +++ b/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-benchmark-profile/SKILL.md @@ -26,6 +26,9 @@ Before running any benchmark, profiler, or kernel-validation command: - set `SGLANG_DIFFUSION_SYNC_STAGE_PROFILING=1` when comparing stage-level denoise/decode timings; the preset helper sets it by default unless the caller explicitly overrides it +- for downloaded checkpoints, use the preset helper's task-owned + `--model-cache-root` together with `--cleanup-model-cache`; verify the JSONL + ledger reports zero residual weight files before moving to the next model - choose idle GPU(s) before starting perf work ## Native Backend Gate @@ -45,10 +48,10 @@ If any benchmark, perf-dump, or `torch.profiler` command prints one of those sig ## Main Reference -- [benchmark-and-profile.md](benchmark-and-profile.md) — canonical denoise benchmark, perf dump, and `torch.profiler` workflow; uses checked-in nightly-aligned presets plus current-source extras such as MiniMax-H3 joint video/audio T2VA, FLUX.2 Klein, Cosmos3, Ideogram4, ERNIE/GLM/SANA image models, FastWan2.2, `LTX-2.3` one-stage/two-stage/HQ, HunyuanVideo, MOVA, Helios, JoyAI/FireRed image edit, and Hunyuan3D shape +- [benchmark-and-profile.md](benchmark-and-profile.md) — canonical denoise benchmark, perf dump, and `torch.profiler` workflow; uses checked-in nightly-aligned presets plus current-source extras such as LongCat-Image, SANA-Video, LingBot Video MoE, Cosmos3 Edge/distilled, LTX-2.5 and its diffusion decoder, MiniMax-H3, FLUX.2 Klein, Ideogram4, ERNIE/GLM/SANA image models, FastWan2.2, `LTX-2.3`, HunyuanVideo, MOVA, Helios, image edit, and Hunyuan3D shape - [existing-fast-paths.md](existing-fast-paths.md) — map bottlenecks to existing fused kernels, packed QKV paths, fused `QK norm + RoPE`, distributed overlap patterns, and open optimization PRs before proposing new code - [scripts/diffusion_skill_env.py](scripts/diffusion_skill_env.py) — preflight helper: repo root discovery via `sglang.__file__`, write-access probe, benchmark/profile output directories, idle GPU selection -- [scripts/bench_diffusion_denoise.py](scripts/bench_diffusion_denoise.py) — end-to-end denoise benchmark preset runner via `sglang generate`; supports `--no-torch-compile`, forces the H3 preset to its eager consistency mode, enables synchronized stage attribution for perf dumps, validates nightly preset drift with `--validate-nightly-alignment`, and saves perf dumps by label for `compare_perf.py` +- [scripts/bench_diffusion_denoise.py](scripts/bench_diffusion_denoise.py) — end-to-end denoise benchmark preset runner via `sglang generate`; defaults to eager, supports opt-in `--torch-compile`, forces H3 to its eager consistency mode, enables synchronized stage attribution, validates nightly preset drift, and can clean an isolated model cache in a `finally` block with a JSONL ledger ## Opportunity Discovery Rule @@ -59,6 +62,8 @@ Always rule out these existing families first: - LTX upsampler GroupNorm+SiLU - Z-Image bf16-native Triton RMSNorm scale/tanh-residual modulation - SANA packed self-attention Q/K/V and cross-attention K/V GEMMs +- SANA-Video reuse of SANA's bit-exact bias/activation, residual-gate, and + LayerNorm-modulation fast paths before adding video-only kernels - MiniMax-H3 indexed modulation, fused QK norm + RoPE, packed Ulysses QKV, USP relayout, and batched TP AdaLN collectives - bit-exact diffusion adaLN modulation and fused LayerNorm + modulation for @@ -68,6 +73,8 @@ Always rule out these existing families first: - fused diffusion `QK norm + RoPE` - LTX2 split RoPE - LTX2 residual-gate add +- LTX-2.5 diffusion-decoder NATTEN selection before interpreting a + FlexAttention fallback trace - varlen USP attention pack/scatter - NVFP4 / Nunchaku packed QKV - Nunchaku fused GELU MLP @@ -77,10 +84,9 @@ Always rule out these existing families first: - breakable CUDA graph capture for supported fixed-resolution pipelines - dual-stream diffusion execution -If the user explicitly requires `torch.compile` to stay off, do not use the -default benchmark preset invocation unchanged. Either pass the checked-in -benchmark helper its no-compile switch or run the equivalent manual command -without `--enable-torch-compile`. +The checked-in helper defaults to eager. Use `--torch-compile` only for a +controlled comparator, never for the eager ground truth. The legacy +`--no-torch-compile` spelling remains accepted but is redundant. MiniMax-H3 is always an eager consistency case on current main. Use `--model minimax-h3-t2va`; its preset writes the H3 request fields through a diff --git a/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-benchmark-profile/benchmark-and-profile.md b/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-benchmark-profile/benchmark-and-profile.md index 98df19ca5..ea03164d3 100644 --- a/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-benchmark-profile/benchmark-and-profile.md +++ b/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-benchmark-profile/benchmark-and-profile.md @@ -135,20 +135,32 @@ PYTHONPATH=python python3 "$BENCH_PY" \ --output-dir "${BENCH_DIR}" ``` +The helper defaults to eager. Add `--torch-compile` only for a labeled compile +control. `--no-torch-compile` remains accepted for compatibility but is no +longer required. + The helper sets `SGLANG_DIFFUSION_SYNC_STAGE_PROFILING=1` for accurate stage attribution. Set it to `0` explicitly only when collecting an e2e-only run and do not compare its per-stage values with synchronized results. -Keep `torch.compile` off when the task requires it: +For downloaded checkpoints, isolate and clean the model cache after the preset +finishes. Cleanup also runs after an error or interruption, and appends a JSONL +record with pre/post byte and weight-file counts: ```bash +MODEL_CACHE_ROOT=/path/to/task-owned/model-caches PYTHONPATH=python python3 "$BENCH_PY" \ - --model flux \ + --model longcat-image \ --label baseline \ --output-dir "${BENCH_DIR}" \ - --no-torch-compile + --model-cache-root "${MODEL_CACHE_ROOT}" \ + --cleanup-model-cache ``` +The helper refuses to reuse an existing per-run cache directory and never +redirects `SGLANG_CACHE_DIR`, so compiled kernel caches remain separate. Never +point this option at a shared Hugging Face or ModelScope cache. + Run the `LTX-2.3` one-stage skill preset: ```bash @@ -177,7 +189,7 @@ PYTHONPATH=python python3 "$BENCH_PY" \ ``` Run the current-source MiniMax-H3 T2VA preset. The helper forces eager mode -for this model even when its global compile default is enabled: +for this model even when `--torch-compile` is requested: ```bash export CUDA_VISIBLE_DEVICES=$(python3 "$ENV_PY" print-idle-gpus --count 4) @@ -215,8 +227,8 @@ Use the preset categories this way: | Preset | Model | Nightly | Notes | | --- | --- | --- | --- | -| `flux` | `black-forest-labs/FLUX.1-dev` | Yes: `flux1_dev_t2i_1024` | Prompt, 1024x1024, seed 42, 2 GPUs, TP size 2, `--dit-layerwise-offload false`; no explicit steps/guidance override | -| `flux2` | `black-forest-labs/FLUX.2-dev` | Yes: `flux2_dev_t2i_1024` | Prompt, 1024x1024, seed 42, 2 GPUs, TP size 2, `--dit-layerwise-offload false`; no explicit steps/guidance override | +| `flux` | `black-forest-labs/FLUX.1-dev` | Yes: `flux1_dev_t2i_1024` | Prompt, 1024x1024, seed 42, 2 GPUs, TP size 2, resident DiT; no explicit steps/guidance override | +| `flux2` | `black-forest-labs/FLUX.2-dev` | Yes: `flux2_dev_t2i_1024` | Prompt, 1024x1024, seed 42, 2 GPUs, TP size 2, resident DiT; no explicit steps/guidance override | | `qwen` | `Qwen/Qwen-Image-2512` | Yes: `qwen_image_2512_t2i_1024` | Prompt, 1024x1024, seed 42, 2 GPUs, TP size 2; no explicit steps/guidance override | | `qwen-edit` | `Qwen/Qwen-Image-Edit-2511` | Yes: `qwen_image_edit_2511` | Uses the nightly cat image and edit prompt, 2 GPUs, TP size 2 | | `zimage` | `Tongyi-MAI/Z-Image-Turbo` | Yes: `zimage_turbo_t2i_1024` | Prompt, 1024x1024, seed 42, 2 GPUs, TP size 2; no explicit steps/guidance override | @@ -226,7 +238,14 @@ Use the preset categories this way: | `ideogram4-fp8` | `ideogram-ai/ideogram-4-fp8` | Yes: `ideogram4_fp8_t2i_2gpu` | Prompt, 1024x1024, seed 42, 2 GPUs, TP size 2, FlashAttention backend; sampling preset owns steps/guidance | | `cosmos3-super-t2v` | `nvidia/Cosmos3-Super` | Yes: `cosmos3_super_t2v_2gpu` | Prompt, 1280x720, 81 frames, seed 42, 2 GPUs, TP size 2, guardrails disabled for benchmark isolation | | `wan-i2v` | `Wan-AI/Wan2.2-I2V-A14B-Diffusers` | Yes: `wan22_i2v_a14b_720p` | Nightly cat image and motion prompt, 1280x720, 81 frames, 4 GPUs, CFG parallel, Ulysses degree 2, text encoder CPU offload and pinned CPU memory | -| `minimax-h3-t2va` | `MiniMaxAI/MiniMax-H3` | No | Current-source H3 FL2VA-partition T2VA baseline: 1344x768 resolved canvas, 5 seconds / 124 frames at 24 fps, 50 joint video-audio steps, 4 GPUs, TP2 + Ulysses2, eager BF16/FP32. The helper writes H3's `task`, `conditions`, `target`, and audio/video flow shifts to a generated config. | +| `minimax-h3-t2va` | `MiniMaxAI/MiniMax-H3` | Yes: `minimax_h3_t2va_5s` | H3 FL2VA-partition T2VA baseline: 1344x768 resolved canvas, 5 seconds / 124 frames at 24 fps, 50 joint video-audio steps, 4 GPUs, TP2 + Ulysses2, eager BF16/FP32. The helper writes H3's request contract to a generated config. | +| `longcat-image` | `meituan-longcat/LongCat-Image` | No | Eager DiT baseline at 1024x1024, 50 steps, guidance 4.5; prompt rewrite is disabled so Qwen2.5-VL does not contaminate the DiT A/B. | +| `sana-video` | `Efficient-Large-Model/SANA-Video_2B_480p_diffusers` | No | CI-sized eager T2V baseline: 832x480, 17 frames, 8 steps, guidance 6.0. | +| `lingbot-video-moe` | `robbyant/lingbot-video-moe-30b-a3b` | No | One-GPU eager baseline using the CI structured-JSON caption, 384x640, 17 frames, 12 steps, and text-encoder CPU offload. | +| `cosmos3-edge-t2i` | `nvidia/Cosmos3-Edge` | No | One-GPU eager T2I baseline at Edge's native 640x640 shape, 35 steps, guidance 7.0. | +| `cosmos3-super-t2i-distilled` | `nvidia/Cosmos3-Super-Text2Image-4Step` | No | Four-GPU eager distilled T2I baseline. The checkpoint owns its fixed sigma schedule; the preset does not override the step count. | +| `ltx25` | `Lightricks/LTX-2.5-Diffusers` | No | One-stage distilled eager baseline at 960x544, 121 frames, 8 steps, guidance 1.0. | +| `ltx25-diffusion-decoder` | `Lightricks/LTX-2.5-Diffusers` | No | Same fixed DiT workload with `--use-diffusion-decoder`; attribute decoder time separately and confirm NATTEN `na3d` is active. | | `ltx2` | `Lightricks/LTX-2` | No | Current-source two-stage LTX-2 preset with 2 GPUs, CFG parallel, 768x512, 121 frames | | `qwen-image` | `Qwen/Qwen-Image` | No | Current-source extra covering the base Qwen-Image native path, separate from the nightly `Qwen-Image-2512` case | | `qwen-edit-2509` | `Qwen/Qwen-Image-Edit-2509` | No | Current-source extra for the pre-2511 edit-plus path; uses the cat image, 1024x1024 | @@ -658,4 +677,5 @@ This skill intentionally stops here. It tells you whether you are looking at: - [ ] one representative `torch.profiler` trace saved - [ ] hotspot classified against `existing-fast-paths.md` - [ ] reference image or video checked for correctness +- [ ] task-owned checkpoint cache cleaned and ledger shows zero residual weight files - [ ] any remaining kernel work handed off with perf/profile evidence attached diff --git a/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-benchmark-profile/existing-fast-paths.md b/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-benchmark-profile/existing-fast-paths.md index 6b6f55904..64077e801 100644 --- a/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-benchmark-profile/existing-fast-paths.md +++ b/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-benchmark-profile/existing-fast-paths.md @@ -32,6 +32,12 @@ framework-specific optimization workflow. - `python/sglang/kernels/ops/diffusion/layout/usp_relayout_jit.py` - `python/sglang/multimodal_gen/runtime/layers/usp.py` - `python/sglang/multimodal_gen/runtime/models/dits/minimax_h3.py` +- `python/sglang/multimodal_gen/runtime/models/dits/longcat_image.py` +- `python/sglang/multimodal_gen/runtime/models/dits/sana_video.py` +- `python/sglang/multimodal_gen/runtime/models/dits/lingbot_video_moe.py` +- `python/sglang/multimodal_gen/runtime/models/decoders/ltx_2_5_diffusion_decoder.py` +- `python/sglang/multimodal_gen/runtime/layers/moe.py` +- `python/sglang/srt/layers/moe/topk.py` - `python/sglang/kernels/ops/diffusion/modulate/residual_gate_add_jit.py` - `python/sglang/kernels/jit/csrc/diffusion/residual_gate_add.cuh` - `python/sglang/kernels/ops/diffusion/layout/varlen_pack_pad_triton.py` @@ -289,6 +295,28 @@ framework-specific optimization workflow. - Scope: this is a mainline SANA model fast path. Query projection in cross-attention remains separate because it uses denoising hidden states, while K/V share step-invariant encoder hidden states. - Workflow rule: if a SANA trace shows separate self-attention `to_q`, `to_k`, `to_v` GEMMs, or separate cross-attention `to_k` and `to_v` GEMMs, treat that as a regressed existing packed-projection path before proposing a new GEMM fusion. +**Recent Model Audit Boundaries** + +- LongCat-Image currently has split image/text QKV projections and performs + joint-stream `cat`/split inside each single block. Do not misclassify those + as a missed existing packed path; they are model-local structural + opportunities that need their own weight-loader and parity coverage. +- SANA-Video already packs self QKV and cross KV. Its conv/modulation formulas + mirror SANA, but it does not yet call SANA's bit-exact bias-SiLU, bias-GLU, + residual-gate, LayerNorm-modulation, or one-time contiguous-layout helpers. + Reuse or extract those helpers before authoring a video-only kernel. +- LingBot Video MoE's router implements sigmoid+bias grouped top-k in + `multimodal_gen/runtime/layers/moe.py`. Check parameter and output-order + compatibility with `srt/layers/moe/topk.py::biased_grouped_topk` before + writing a new router kernel. +- LTX-2.5 reuses the mature LTX-2 DiT paths. Treat the optional diffusion + decoder separately: confirm NATTEN `na3d` is active, then inspect its + per-block 3D RoPE construction and split QKV/SwiGLU projections. +- Cosmos3 Edge inherits the existing Cosmos3 attention-prep fusions. Profile + the dense squared-ReLU MLP before proposing another Cosmos kernel, and do not + repeat the closed experimental Cosmos BCG direction without solving its + model-state lifecycle problem. + **Common Entry Points in Diffusion Models** - AdaLN modulation: `LayerNormScaleShift`, `RMSNormScaleShift`, `ScaleResidual*` in `layernorm.py`. - Bit-exact adaLN modulation / LayerNorm folding: `modulate_scale_shift` and @@ -339,9 +367,10 @@ framework-specific optimization workflow. - Breakable CUDA graph: `runtime/breakable_cuda_graph/runner.py` captures fixed-resolution DiT segments around eager attention/collectives for supported pipelines. It is mutually exclusive with `torch.compile` and - Cache-DiT, requires every served resolution in `--warmup-resolutions`, and - uses `--bcg-text-buckets` for prompt signatures. Check this path before - proposing a second graph-capture mechanism for launch-bound traces. + Cache-DiT. The model's default resolution is captured automatically; put + every additional served resolution in `--warmup-resolutions`, and use + `--bcg-text-buckets` for prompt signatures. Check this path before proposing + a second graph-capture mechanism for launch-bound traces. - Dual-stream diffusion models: `use_dual_stream = True` in models such as `hunyuan3d.py` is an existing overlap family. - Workflow rule: if a hotspot is communication-heavy, rule out these in-repo overlap families before proposing a brand new overlap design. @@ -361,6 +390,9 @@ relying on any file path, flag, or claim about whether the work has merged. - #20429 Qwen-Image layernorm and `fuse_scale_shift_gate_select01` work. - #20530 MOVA fused RMSNorm + interleaved RoPE. - #29361 LTX2 residual-gate CUDA fast path for `residual + update * gate`. + - #34172 LTX2 quality-high fusion; #34305/#34314 Ideogram eager fusions. + - #34584 Wan TI2V modulation/RoPE; #34616 FLUX2; #34617 Hunyuan; + #34619 GLM; #34620 ERNIE; #34928 SANA; #34932 Cosmos3. - VAE and decode-side acceleration: - #22531 LTX2 parallel VAE support and #20927 batched tiled VAE decode (draft). - Attention, communication, and runtime scheduling: @@ -375,6 +407,9 @@ relying on any file path, flag, or claim about whether the work has merged. - #20447 TeaCache support for GLM-Image, Qwen-Image, and related models. - #19516 Qwen-Image CUDA Graph. - #21912 Z-Image Turbo FP8 full quantization and CUDA Graph. + - #34174 automatic default-resolution BCG warmup; #34210 Z-Image BCG + correctness; #34929 LTX2.3 BCG. #34618 is a closed Cosmos BCG experiment, + not a reusable mainline fast path. **Constraints and Fallbacks** - `scale_shift` Triton requires CUDA + contiguous `x`. NPU swaps to native. diff --git a/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-benchmark-profile/scripts/bench_diffusion_denoise.py b/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-benchmark-profile/scripts/bench_diffusion_denoise.py index d2e31843d..382ae44e2 100755 --- a/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-benchmark-profile/scripts/bench_diffusion_denoise.py +++ b/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-benchmark-profile/scripts/bench_diffusion_denoise.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 """ End-to-end denoise-stage benchmark presets for SGLang Diffusion. @@ -12,6 +13,12 @@ Usage: # Tag the run for later compare_perf.py usage python3 python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-benchmark-profile/scripts/bench_diffusion_denoise.py --model flux --label tuned + # Opt in to a compile control (presets are eager by default) + python3 python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-benchmark-profile/scripts/bench_diffusion_denoise.py --model flux --torch-compile + + # Clean an isolated model cache even if the run fails or is interrupted + python3 python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-benchmark-profile/scripts/bench_diffusion_denoise.py --model longcat-image --model-cache-root /task/model-caches --cleanup-model-cache + # All preset models python3 python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-benchmark-profile/scripts/bench_diffusion_denoise.py --all @@ -33,17 +40,17 @@ import argparse import json import os import shlex +import shutil import subprocess import sys import time from pathlib import Path -from typing import Optional SCRIPT_DIR = Path(__file__).resolve().parent if str(SCRIPT_DIR) not in sys.path: sys.path.insert(0, str(SCRIPT_DIR)) -from diffusion_skill_env import ( # noqa: E402 +from diffusion_skill_env import ( ensure_dir, get_assets_dir, get_output_dir, @@ -69,6 +76,15 @@ DIFFUSERS_FALLBACK_SIGNALS = ( ) CATALOG_TABLE_WIDTH = 140 RESULTS_TABLE_WIDTH = 105 +MODEL_CACHE_MARKER = ".sglang-diffusion-benchmark-cache" +MODEL_WEIGHT_SUFFIXES = { + ".bin", + ".ckpt", + ".gguf", + ".pt", + ".pth", + ".safetensors", +} NIGHTLY_PRESET_ORDER = ( "flux", "flux2", @@ -81,6 +97,58 @@ NIGHTLY_PRESET_ORDER = ( "ideogram4-fp8", "cosmos3-super-t2v", "wan-i2v", + "minimax-h3-t2va", +) + +LINGBOT_VIDEO_PROMPT = json.dumps( + { + "comprehensive_description": { + "scene_content_description": ( + "A small silver robot arm on a white table slowly reaches " + "toward a red cube. The background is a plain, softly lit " + "laboratory wall." + ), + "camera_movement_description": ( + "The camera is static at eye level, medium shot, with the " + "robot arm centered and in sharp focus." + ), + }, + "camera_info": { + "color": "Neutral", + "frame_size": "Medium", + "shot_type_angle": "Eye level", + "lens_size": "Medium", + "composition": "Center", + "lighting": "Soft light", + "lighting_type": "Artificial light", + }, + "world_knowledge": [], + "prominent_elements": [ + { + "name": "robot arm", + "description": "A small silver robot arm with a two-finger gripper.", + "actions": [ + { + "timestamp": "[0.0s - 1.0s]", + "action": "reaches toward the red cube", + } + ], + "location": "center of the frame", + "relative_size": "dominant", + "shape_and_color": "articulated silver metal arm", + "texture": "brushed metal", + "appearance_details": "two-finger gripper, visible joints", + "relationship": "reaching toward the red cube on the table", + "orientation": "upright, base on the table", + "pose": "reaching", + "expression": "", + "clothing": "", + "gender": "", + "skin_tone_and_texture": "", + } + ], + }, + separators=(",", ":"), ) # --------------------------------------------------------------------------- @@ -100,8 +168,7 @@ MODELS = { "--height=1024", "--num-gpus=2", "--tp-size=2", - "--dit-layerwise-offload", - "false", + "--component-residency=dit=resident", ], }, # 2. Nightly: flux2_dev_t2i_1024 @@ -114,8 +181,7 @@ MODELS = { "--height=1024", "--num-gpus=2", "--tp-size=2", - "--dit-layerwise-offload", - "false", + "--component-residency=dit=resident", ], }, # 3. Nightly: qwen_image_2512_t2i_1024 @@ -248,11 +314,12 @@ MODELS = { "--pin-cpu-memory", ], }, - # Source-tracked extras from current registry / GPU test coverage. + # 12. Nightly: minimax_h3_t2va_5s # MiniMax-H3 owns its temporal canvas through target.duration_seconds, so # the model-specific sampling fields are passed through --config instead # of generic --width/--height/--num-frames flags. "minimax-h3-t2va": { + "nightly_case_id": "minimax_h3_t2va_5s", "path": "MiniMaxAI/MiniMax-H3", "prompt": "At night, while their owner sleeps in a bedroom, three cats march in loudly playing tiny brass instruments, then abruptly file out.", "seed": 1101, @@ -280,6 +347,115 @@ MODELS = { # torch.compile changes numerical output, so never add the global # helper default --enable-torch-compile flag for this preset. "force_eager": True, + "nightly_cli_ignored": { + "width", + "height", + "num-frames", + "fps", + "num-inference-steps", + }, + }, + # Source-tracked extras from current registry / GPU test coverage. + "longcat-image": { + "path": "meituan-longcat/LongCat-Image", + "prompt": "A red panda reading a book beside a sunlit window.", + "extra_args": [ + "--width=1024", + "--height=1024", + "--num-inference-steps=50", + "--guidance-scale=4.5", + "--enable-prompt-rewrite=false", + "--performance-mode=manual", + ], + }, + "sana-video": { + "path": "Efficient-Large-Model/SANA-Video_2B_480p_diffusers", + "prompt": "A curious raccoon walks through a sunlit forest. motion score: 30.", + "extra_args": [ + "--width=832", + "--height=480", + "--num-frames=17", + "--fps=16", + "--num-inference-steps=8", + "--guidance-scale=6.0", + "--performance-mode=manual", + ], + }, + "lingbot-video-moe": { + "path": "robbyant/lingbot-video-moe-30b-a3b", + "prompt": LINGBOT_VIDEO_PROMPT, + "seed": 0, + "extra_args": [ + "--width=384", + "--height=640", + "--num-frames=17", + "--fps=16", + "--num-inference-steps=12", + "--text-encoder-cpu-offload", + "--performance-mode=manual", + ], + }, + "cosmos3-edge-t2i": { + "path": "nvidia/Cosmos3-Edge", + "prompt": "A warehouse robot folds a blue cloth on a clean workbench.", + "seed": 0, + "env": { + "SGLANG_DISABLE_COSMOS3_GUARDRAILS": "1", + }, + "extra_args": [ + "--width=640", + "--height=640", + "--num-frames=1", + "--num-inference-steps=35", + "--guidance-scale=7.0", + "--performance-mode=manual", + ], + }, + "cosmos3-super-t2i-distilled": { + "path": "nvidia/Cosmos3-Super-Text2Image-4Step", + "prompt": "A warehouse robot folds a blue cloth on a clean workbench.", + "seed": 0, + "env": { + "SGLANG_DISABLE_COSMOS3_GUARDRAILS": "1", + }, + "extra_args": [ + "--width=640", + "--height=640", + "--num-frames=1", + "--guidance-scale=1.0", + "--num-gpus=4", + "--tp-size=4", + "--performance-mode=manual", + ], + }, + "ltx25": { + "path": "Lightricks/LTX-2.5-Diffusers", + "prompt": "A cat and a dog baking a cake together in a kitchen.", + "extra_args": [ + "--pipeline-class-name=LTX2Pipeline", + "--width=960", + "--height=544", + "--num-frames=121", + "--fps=24", + "--num-inference-steps=8", + "--guidance-scale=1.0", + "--performance-mode=manual", + ], + }, + "ltx25-diffusion-decoder": { + "path": "Lightricks/LTX-2.5-Diffusers", + "prompt": "A cat and a dog baking a cake together in a kitchen.", + "extra_args": [ + "--pipeline-class-name=LTX2Pipeline", + "--width=960", + "--height=544", + "--num-frames=121", + "--fps=24", + "--num-inference-steps=8", + "--guidance-scale=1.0", + "--use-diffusion-decoder", + "--performance-mode=manual", + ], }, "ltx2": { "path": "Lightricks/LTX-2", @@ -618,6 +794,123 @@ def model_nightly_case_id(model_key: str) -> str: return MODELS[model_key].get("nightly_case_id", "-") +def _safe_cache_component(value: str) -> str: + component = "".join( + character if character.isalnum() or character in "-_." else "_" + for character in value + ).strip(".") + if not component: + raise ValueError(f"Cannot derive a cache directory name from {value!r}") + return component + + +def _prepare_model_cache(cache_root: Path, model_key: str, label: str) -> Path: + cache_root = cache_root.expanduser().resolve() + unsafe_roots = {Path("/"), Path.home().resolve(), REPO_ROOT.resolve()} + if cache_root in unsafe_roots: + raise ValueError( + "Refusing to use a broad or shared directory as the isolated model " + f"cache root: {cache_root}" + ) + + cache_root.mkdir(parents=True, exist_ok=True) + marker = cache_root / MODEL_CACHE_MARKER + if not marker.exists(): + marker.write_text( + "Owned by bench_diffusion_denoise.py. Only generated child caches " + "may be removed.\n", + encoding="utf-8", + ) + + cache_dir = cache_root / ( + f"{_safe_cache_component(model_key)}-{_safe_cache_component(label)}" + ) + if cache_dir.exists(): + raise FileExistsError( + "The isolated model cache already exists. Refusing to reuse or " + f"delete it without inspection: {cache_dir}" + ) + cache_dir.mkdir() + return cache_dir + + +def _model_cache_env(cache_dir: Path) -> dict[str, str]: + huggingface_root = cache_dir / "huggingface" + huggingface_hub = huggingface_root / "hub" + return { + "HF_HOME": str(huggingface_root), + "HF_ASSETS_CACHE": str(huggingface_root / "assets"), + "HF_HUB_CACHE": str(huggingface_hub), + "HF_MODULES_CACHE": str(huggingface_root / "modules"), + "HF_XET_CACHE": str(huggingface_root / "xet"), + "HUGGINGFACE_HUB_CACHE": str(huggingface_hub), + "DIFFUSERS_CACHE": str(huggingface_hub), + "TRANSFORMERS_CACHE": str(huggingface_hub), + "MODELSCOPE_CACHE": str(cache_dir / "modelscope"), + "MODELSCOPE_MODULES_CACHE": str(cache_dir / "modelscope" / "modules"), + } + + +def _cache_stats(cache_dir: Path) -> dict[str, int]: + file_count = 0 + weight_file_count = 0 + total_bytes = 0 + if not cache_dir.exists(): + return { + "file_count": 0, + "weight_file_count": 0, + "total_bytes": 0, + } + + for entry in cache_dir.rglob("*"): + if not (entry.is_file() or entry.is_symlink()): + continue + file_count += 1 + total_bytes += entry.lstat().st_size + if entry.suffix.lower() in MODEL_WEIGHT_SUFFIXES: + weight_file_count += 1 + return { + "file_count": file_count, + "weight_file_count": weight_file_count, + "total_bytes": total_bytes, + } + + +def _cleanup_model_cache( + cache_root: Path, + cache_dir: Path, + ledger_path: Path, + model_key: str, + label: str, + exit_reason: str, +) -> dict[str, object]: + cache_root = cache_root.expanduser().resolve() + cache_dir = cache_dir.resolve() + if not (cache_root / MODEL_CACHE_MARKER).is_file(): + raise RuntimeError(f"Missing isolated-cache ownership marker: {cache_root}") + if cache_dir.parent != cache_root: + raise RuntimeError( + f"Refusing to remove cache outside the isolated root: {cache_dir}" + ) + + before = _cache_stats(cache_dir) + shutil.rmtree(cache_dir) + after = _cache_stats(cache_dir) + record: dict[str, object] = { + "model": model_key, + "label": label, + "exit_reason": exit_reason, + "cache_dir": str(cache_dir), + "cleaned_at_unix_s": time.time(), + "before": before, + "after": after, + } + ledger_path.parent.mkdir(parents=True, exist_ok=True) + with ledger_path.open("a", encoding="utf-8") as ledger: + ledger.write(json.dumps(record, sort_keys=True) + "\n") + return record + + def _parse_cli_args(args: list[str]) -> dict[str, object]: parsed: dict[str, object] = {} i = 0 @@ -669,7 +962,7 @@ def _expected_nightly_cli_args(case: dict) -> dict[str, str]: # switch. It is not a valid ``sglang generate`` flag after the # warmup-mode migration, so exclude both spellings from preset drift # validation. - if flag in {"enable-torch-compile", "warmup", "warmup-mode"}: + if flag in {"warmup", "warmup-mode"}: continue expected[flag] = _normalize_cli_value(value) @@ -716,11 +1009,32 @@ def validate_nightly_alignment() -> int: if preset.get("env", {}) != case["frameworks"]["sglang"].get("extra_env", {}): errors.append(f"{model_key}: environment differs") + if "config_overrides" in preset: + expected_config = { + key: value + for key, value in case.get("sglang_request_extra", {}).items() + if value is not None + } + if "num_inference_steps" in case: + expected_config["num_inference_steps"] = case["num_inference_steps"] + if preset["config_overrides"] != expected_config: + errors.append( + f"{model_key}: generated request config differs\n" + f" skill={preset['config_overrides']}\n" + f" ci={expected_config}" + ) + + ignored_args = set(preset.get("nightly_cli_ignored", set())) actual_args = { key: _normalize_cli_value(value) for key, value in _parse_cli_args(preset["extra_args"]).items() + if key not in ignored_args + } + expected_args = { + key: value + for key, value in _expected_nightly_cli_args(case).items() + if key not in ignored_args } - expected_args = _expected_nightly_cli_args(case) if actual_args != expected_args: errors.append( f"{model_key}: CLI args differ\n" @@ -761,12 +1075,12 @@ def print_model_catalog(): def build_sglang_cmd( model_key: str, - perf_dump_path: Optional[str] = None, + perf_dump_path: str | None = None, warmup: bool = True, - torch_compile: bool = True, + torch_compile: bool = False, seed: int = 42, save_output: bool = True, - artifact_dir: Optional[Path] = None, + artifact_dir: Path | None = None, ) -> list[str]: """ Build the `sglang generate` command for the given model. @@ -818,12 +1132,13 @@ def build_sglang_cmd( return cmd -def run_benchmark_once( +def _run_benchmark_once_impl( model_key: str, label: str, output_dir: Path, warmup: bool = True, - torch_compile: bool = True, + torch_compile: bool = False, + model_cache_dir: Path | None = None, ) -> dict: """Run a single benchmark pass and return results dict.""" perf_path = output_dir / f"{model_key}_{label}.json" @@ -846,6 +1161,8 @@ def run_benchmark_once( cfg = MODELS[model_key] for key, value in cfg.get("env", {}).items(): env.setdefault(key, str(value)) + if model_cache_dir is not None: + env.update(_model_cache_env(model_cache_dir)) if env.get("HF_TOKEN") and not env.get("HUGGINGFACE_HUB_TOKEN"): env["HUGGINGFACE_HUB_TOKEN"] = env["HF_TOKEN"] @@ -883,10 +1200,20 @@ def run_benchmark_once( ) fallback_detected = False assert process.stdout is not None - for line in process.stdout: - print(line, end="") - if any(signal in line.lower() for signal in DIFFUSERS_FALLBACK_SIGNALS): - fallback_detected = True + try: + for line in process.stdout: + print(line, end="") + if any(signal in line.lower() for signal in DIFFUSERS_FALLBACK_SIGNALS): + fallback_detected = True + except BaseException: + if process.poll() is None: + process.terminate() + try: + process.wait(timeout=10) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + raise returncode = process.wait() elapsed = time.time() - t0 @@ -925,10 +1252,7 @@ def run_benchmark_once( if ( isinstance(step_name, str) and step.get("duration_ms") is not None - and ( - step_name.endswith("DenoisingStage") - or step_name.endswith("RefinementStage") - ) + and step_name.endswith(("DenoisingStage", "RefinementStage")) and "BeforeDenoisingStage" not in step_name ): denoise_stage_total_ms += float(step["duration_ms"]) @@ -957,12 +1281,63 @@ def run_benchmark_once( peak_memory_gb = candidate metrics["peak_memory_gb"] = peak_memory_gb - except Exception as e: + except (AttributeError, OSError, TypeError, ValueError) as e: print(f" Warning: could not parse perf dump: {e}") return metrics +def run_benchmark_once( + model_key: str, + label: str, + output_dir: Path, + warmup: bool = True, + torch_compile: bool = False, + model_cache_root: Path | None = None, + cleanup_model_cache: bool = False, + cleanup_ledger_path: Path | None = None, +) -> dict: + """Run one preset and optionally clean its task-owned model cache.""" + cache_dir = None + exit_reason = "error" + if model_cache_root is not None: + cache_dir = _prepare_model_cache(model_cache_root, model_key, label) + + try: + result = _run_benchmark_once_impl( + model_key, + label, + output_dir, + warmup=warmup, + torch_compile=torch_compile, + model_cache_dir=cache_dir, + ) + exit_reason = "error" if result.get("error") else "success" + return result + except KeyboardInterrupt: + exit_reason = "interrupted" + raise + finally: + if cleanup_model_cache and cache_dir is not None: + assert model_cache_root is not None + ledger_path = cleanup_ledger_path or output_dir / "cleanup.jsonl" + record = _cleanup_model_cache( + model_cache_root, + cache_dir, + ledger_path, + model_key, + label, + exit_reason, + ) + before = record["before"] + assert isinstance(before, dict) + print( + " Cleaned isolated model cache: " + f"{before['total_bytes']} bytes, " + f"{before['weight_file_count']} weight files; ledger={ledger_path}" + ) + + def print_results_table(results: list[dict]): """Print a compact table for one or more benchmark runs.""" print() @@ -1032,10 +1407,37 @@ def main(): help="Directory for perf dump JSON files", ) parser.add_argument("--no-warmup", action="store_true", help="Skip warmup") - parser.add_argument( + compile_group = parser.add_mutually_exclusive_group() + compile_group.add_argument( + "--torch-compile", + action="store_true", + help="Opt in to a torch.compile comparison. Presets run eager by default.", + ) + compile_group.add_argument( "--no-torch-compile", action="store_true", - help="Keep torch.compile disabled for eager-mode comparisons.", + help="Deprecated compatibility flag; eager is already the default.", + ) + parser.add_argument( + "--model-cache-root", + type=str, + help=( + "Create a new isolated Hugging Face/ModelScope cache below this " + "directory for each model run." + ), + ) + parser.add_argument( + "--cleanup-model-cache", + action="store_true", + help=( + "Remove the task-owned model cache in a finally block and append " + "a cleanup ledger record. Requires --model-cache-root." + ), + ) + parser.add_argument( + "--cleanup-ledger", + type=str, + help="JSONL cleanup ledger path (default: /cleanup.jsonl).", ) args = parser.parse_args() @@ -1050,7 +1452,15 @@ def main(): output_dir = Path(args.output_dir) output_dir.mkdir(parents=True, exist_ok=True) warmup = not args.no_warmup - torch_compile = not args.no_torch_compile + torch_compile = args.torch_compile and not args.no_torch_compile + if args.cleanup_model_cache and not args.model_cache_root: + parser.error("--cleanup-model-cache requires --model-cache-root") + model_cache_root = ( + Path(args.model_cache_root) if args.model_cache_root is not None else None + ) + cleanup_ledger_path = ( + Path(args.cleanup_ledger) if args.cleanup_ledger is not None else None + ) models_to_run = list(MODELS.keys()) if args.all else [args.model or "flux"] results = [] @@ -1063,6 +1473,9 @@ def main(): output_dir, warmup=warmup, torch_compile=torch_compile, + model_cache_root=model_cache_root, + cleanup_model_cache=args.cleanup_model_cache, + cleanup_ledger_path=cleanup_ledger_path, ) ) diff --git a/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-performance/SKILL.md b/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-performance/SKILL.md index 99fd15bca..1d1e21dd4 100644 --- a/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-performance/SKILL.md +++ b/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-performance/SKILL.md @@ -11,6 +11,9 @@ Before running any `sglang generate` command below inside the diffusion containe - use `python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-benchmark-profile/scripts/diffusion_skill_env.py` to derive the repo root, verify write access, and choose idle GPU(s) - export `HF_TOKEN` first when the selected model lives in a gated Hugging Face repo such as `black-forest-labs/FLUX.*` - export `FLASHINFER_DISABLE_VERSION_CHECK=1` +- when a run downloads weights, use a task-owned cache and delete that model's + cache after its eager/compile/BCG/profile group finishes; the benchmark skill + provides `--model-cache-root --cleanup-model-cache` plus a cleanup ledger - `cd` to the repo root resolved from `sglang.__file__` ## Native Backend Gate @@ -33,8 +36,8 @@ These options are intended to preserve output quality. In practice, some paths ( | Option | CLI Flag / Env Var | What It Does | Speedup | Limitations / Notes | |---|---|---|---|---| | **Performance Mode** | `--performance-mode auto\|speed\|memory\|manual` (`--mode` alias) | Applies model-aware residency, FSDP/CFG, and compile defaults without overriding explicit flags. `auto` is the safe default; `speed` favors GPU residency; `memory` favors offload; `manual` leaves performance args explicit. | Fastest way to establish a sensible deployment baseline | `speed` may OOM and enables `torch.compile` only when the model deployment config allows it. Explicit offload/FSDP/parallelism/compile flags win. Use `manual` for controlled A/B benchmarks. | -| **torch.compile** | `--enable-torch-compile` | Applies `torch.compile` to the DiT forward pass, fusing ops and reducing kernel launch overhead. | ~1.2–1.5x on denoising | First request is slow (compilation). May cause minor precision drifts due to [PyTorch issue #145213](https://github.com/pytorch/pytorch/issues/145213). Pair with `--warmup-mode request` for best results. | -| **Breakable CUDA Graph** | `--enable-breakable-cuda-graph --warmup-resolutions ` plus optional `--bcg-text-buckets ...` | Captures fixed-resolution DiT segments while leaving attention/collectives eager, reducing launch overhead on supported pipelines. | Large on launch-bound paths; merged SANA and LTX-2 cases show material e2e gains | Mutually exclusive with `torch.compile` and Cache-DiT; BCG takes priority. Every served resolution must be declared for warmup capture. Current support is model-specific (Ideogram4, LTX-2, MiniMax-H3, Qwen-Image, SANA1.5, Z-Image, GLM-Image); benchmark before keeping it. | +| **torch.compile** | `--enable-torch-compile` | Applies `torch.compile` to the DiT forward pass. Treat it as a measured comparator, not an assumed upgrade. | Model- and shape-dependent; recent B300 coverage found eager or valid BCG faster or within 1% for every valid compile control | First request is slow and some models time out or drift numerically. Keep eager as the ground truth, use a warmup watchdog, and validate the target model. See the [H200/B300 survey](https://github.com/BBuf/how-to-optim-algorithm-in-cuda/issues/21). | +| **Breakable CUDA Graph** | `--enable-breakable-cuda-graph` plus optional `--warmup-resolutions ` and `--bcg-text-buckets ...` | Captures fixed-resolution DiT segments while leaving attention/collectives eager, reducing launch overhead on supported pipelines. | Large on launch-bound paths; merged SANA and LTX-2 cases show material e2e gains | Mutually exclusive with `torch.compile` and Cache-DiT; BCG takes priority. The model's default resolution is captured automatically; declare every additional production resolution. Current support is model-specific (Ideogram4, LTX-2/2.3, MiniMax-H3, Qwen-Image, SANA1.5, Z-Image, GLM-Image); benchmark before keeping it. | | **Warmup** | `--warmup-mode request` | Runs dummy forward passes to warm up CUDA caches, JIT, and `torch.compile`. Eliminates cold-start penalty. | Removes first-request latency spike | Adds startup time. Without `--warmup-resolutions`, warmup happens on first request. | | **Warmup Resolutions** | `--warmup-resolutions 256x256 720x720` | Pre-compiles and warms up specific resolutions at server startup (instead of lazily on first request). | Faster first request per resolution | Each resolution adds to startup time. Serving mode only; useful when you know your target resolutions in advance. | | **Multi-GPU (SP)** | `--num-gpus N --ulysses-degree N` | Sequence parallelism across GPUs. Shards sequence tokens (not frames) to minimize padding. | Near-linear scaling with N GPUs | Requires NCCL; inter-GPU bandwidth matters. `ulysses_degree * ring_degree = sp_degree`. For Wan2.2 video, start by benchmarking pure Ulysses before assuming a mixed Ulysses/Ring layout is fastest. | @@ -364,9 +367,13 @@ Use these as first commands to benchmark, not as universal winners. | Wan2.2 TI2V 5B | 1280x720, 81 frames, 1 GPU | `--enable-torch-compile --warmup-mode request` | Keep the input image and motion prompt fixed when comparing sparse attention or Cache-DiT. | | Wan2.1 / FastWan / TurboWan variants | 480p or 720p video, family defaults | `--enable-torch-compile --warmup-mode request`; add `--ulysses-degree` / CFG parallel only after measuring | Current registry includes Wan2.1, FastWan2.1, FastWan2.2 TI2V, TurboWan2.1, TurboWan2.2 I2V, and Wan2.1-Fun InP. Use the compatibility matrix and benchmark presets before choosing topology. | | Cosmos3 Nano / Super | T2I: 1024x1024 with `--num-frames 1`; T2V/I2V: 480p/720p video | `SGLANG_DISABLE_COSMOS3_GUARDRAILS=1` for benchmark isolation; `--enable-torch-compile --warmup-mode request` | One checkpoint serves T2I/T2V/I2V. Mode is request-driven: `num_frames == 1` means T2I, `--image-path` means I2V. | +| Cosmos3 Edge / distilled Super | Edge T2I: 640x640, 35 steps, 1 GPU; distilled Super T2I: 640x640, fixed 4-step schedule, 4 GPUs | Start eager with `--performance-mode manual`; use `SGLANG_DISABLE_COSMOS3_GUARDRAILS=1` only for benchmark isolation | Edge is trained for 256p/480p shapes. Distilled checkpoints own their sigma schedule and force guidance 1.0; do not override steps or flow shift. Do not retry the closed experimental Cosmos BCG path without a new lifecycle design. | | Ideogram 4 FP8/NVFP4 | 1024x1024, native preset defaults | `--enable-torch-compile --warmup-mode request` | Do not set `--num-inference-steps` or `--guidance-scale` directly unless you also update the Ideogram preset; sampling params derive them from `preset`. | | ERNIE-Image / GLM-Image / SANA / SD3 | 1024-class image, family defaults | `--enable-torch-compile --warmup-mode request`; disable offload only after checking VRAM | Treat these as current native image families. Start with benchmark/profile presets for ERNIE, GLM, and SANA; use registry/config defaults for SD3 unless you add a new preset. | +| LongCat-Image | 1024x1024, 50 steps, guidance 4.5, 1 GPU | `--performance-mode manual --enable-prompt-rewrite false` for a DiT-only eager baseline | Prompt rewriting is enabled by the model defaults and runs a Qwen2.5-VL component. Disable it for kernel A/B, then keep a separate end-to-end recipe with rewriting enabled. | +| SANA-Video | 832x480, 17 frames, 8 steps for CI-sized profiling; 81 frames, 50 steps for release quality | `--performance-mode manual` and eager first | Self QKV and cross KV are already packed. Check SANA's shared bit-exact conv/modulation fast paths and one-time contiguous layout before adding a new kernel. | | LTX-2 / LTX-2.3 | 768x512 or HQ 1920x1088, 121 frames | `--pipeline-class-name LTX2TwoStagePipeline --enable-torch-compile --warmup-mode request`; HQ uses `LTX2TwoStageHQPipeline` | Use benchmark/profile presets for nightly alignment, one-stage, high-resolution stress, and HQ. Device mode choices are `original` and `resident`; `resident` is fastest but uses more VRAM. `snapshot` is a deprecated alias for `original`, so do not use it in new commands. | +| LTX-2.5 | One-stage distilled: 960x544, 121 frames, 8 steps; two-stage: 1920x1088 | `--pipeline-class-name LTX2Pipeline --performance-mode manual`; add `--use-diffusion-decoder` only for the decoder A/B | Benchmark the DiT and optional diffusion decoder as separate stages. Confirm NATTEN `na3d` is active before comparing decoder latency; a FlexAttention fallback is a different backend. Distilled weights run unguided. | | HunyuanVideo | 848x480 or 720p class video | `--text-encoder-cpu-offload --pin-cpu-memory --enable-torch-compile --warmup-mode request` | Check VAE decode separately. GroupNorm+SiLU is default-eligible in mainline when wrapper guards pass; use `bench_group_norm_silu.py` when VAE residual blocks are hot. | | JoyAI-Image-Edit | 1024-class TI2I, 40 steps, guidance 4.0 | `--backend=sglang --num-gpus 2 --enable-cfg-parallel --ulysses-degree 1 --enable-torch-compile --warmup-mode request --dit-layerwise-offload false --dit-cpu-offload false` | Newly supported image-edit path. Keep the input image, prompt, seed, and output size fixed; 2-GPU CFG parallel is the validated H100 starting point. | | FireRed-Image-Edit 1.0 / 1.1 | 1024x1024 image edit, 40 steps, guidance 4.0 | `--backend=sglang --num-gpus 2 --enable-cfg-parallel --ulysses-degree 1 --enable-torch-compile --warmup-mode request --dit-layerwise-offload false --dit-cpu-offload false` | Uses the native `QwenImageEditPlusPipeline` path. 2-GPU CFG parallel is the validated H100 starting point; benchmark 1.0 and 1.1 separately because checkpoint differences can change denoise latency. | @@ -381,12 +388,20 @@ state and the active source tree before relying on any path, flag, or claim about whether the work has merged: - Fusion/kernel: #24025 LTX2 QK norm, #24059 Helios norm modulation, #24117 Z-Image packed QKV, #19488 Wan elementwise cross-block fusion, #19249 Z-Image gate/norm fusion, #20429 Qwen-Image layernorm/modulation, #20530 MOVA RMSNorm+RoPE. +- Recent eager/BCG work: #34172 LTX2 quality-high fusion, #34174 automatic + default-resolution BCG warmup, #34210 Z-Image BCG correctness, #34305/#34314 + Ideogram eager fusions, #34584 Wan TI2V modulation/RoPE, #34616 FLUX2, + #34617 Hunyuan, #34619 GLM, #34620 ERNIE, #34928 SANA, #34929 LTX2.3, + and #34932 Cosmos3. Re-check open/merged state before reusing a path. - VAE/decode: #22531 LTX2 parallel VAE, #20927 batched tiled VAE decode. - Runtime/parallel/cache: #22805 FLUX.2 packed QKV for A2A, #21742 hybrid attention schedule, #24053 USP replicated-prefix fix, #21613 TeaCache refactor, #24227 WanVideo TeaCache fix, #18764 dynamic batching, #24200 disaggregated diffusion. ## Tips -- **Benchmarking**: always use `--warmup-mode request` and look for the line ending with `(with warmup excluded)` for accurate timing. +- **Benchmarking**: establish eager first (`--performance-mode manual`, compile/BCG/cache off), always use `--warmup-mode request`, and look for the line ending with `(with warmup excluded)` for accurate timing. Add compile or BCG as separate labeled controls. +- **Checkpoint cleanup**: finish every variant for one model, then delete only + its task-owned cache and verify the cleanup ledger reports zero residual + weight files. Never point cleanup at a shared Hugging Face or ModelScope cache. - **Preset vs experiment control**: start with `--performance-mode auto` or `speed` for deployment, but use `--performance-mode manual` and pin the relevant residency/parallelism flags for controlled A/B claims. diff --git a/python/sglang/multimodal_gen/test/unit/test_diffusion_benchmark_skill.py b/python/sglang/multimodal_gen/test/unit/test_diffusion_benchmark_skill.py new file mode 100644 index 000000000..bac804d9c --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_diffusion_benchmark_skill.py @@ -0,0 +1,185 @@ +import importlib.util +import json +import sys +import tempfile +import types +import unittest +from pathlib import Path +from unittest.mock import patch + + +def _load_benchmark_module(temp_root: Path): + multimodal_gen_root = Path(__file__).resolve().parents[2] + script_path = ( + multimodal_gen_root + / ".claude" + / "skills" + / "sglang-diffusion-benchmark-profile" + / "scripts" + / "bench_diffusion_denoise.py" + ) + fake_env = types.ModuleType("diffusion_skill_env") + fake_env.ensure_dir = lambda path: ( + Path(path).mkdir(parents=True, exist_ok=True) or Path(path) + ) + fake_env.get_assets_dir = lambda _root: temp_root / "assets" + fake_env.get_output_dir = lambda _kind, _root: temp_root / "outputs" + fake_env.get_repo_root = lambda: temp_root / "repo" + fake_env.pick_idle_gpus = lambda count: list(range(count)) + + spec = importlib.util.spec_from_file_location( + "test_bench_diffusion_denoise", script_path + ) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + with patch.dict(sys.modules, {"diffusion_skill_env": fake_env}): + spec.loader.exec_module(module) + return module + + +class TestDiffusionBenchmarkSkill(unittest.TestCase): + def test_nightly_presets_remain_aligned(self): + with tempfile.TemporaryDirectory() as tmpdir: + module = _load_benchmark_module(Path(tmpdir)) + repo_root = Path(__file__).resolve().parents[5] + module.NIGHTLY_CONFIG_PATH = ( + repo_root + / "scripts" + / "ci" + / "utils" + / "diffusion" + / "comparison_configs.json" + ) + + self.assertEqual(module.validate_nightly_alignment(), 0) + + def test_recent_model_presets_are_eager_by_default(self): + with tempfile.TemporaryDirectory() as tmpdir: + module = _load_benchmark_module(Path(tmpdir)) + + expected = { + "longcat-image", + "sana-video", + "lingbot-video-moe", + "cosmos3-edge-t2i", + "cosmos3-super-t2i-distilled", + "ltx25", + "ltx25-diffusion-decoder", + } + self.assertTrue(expected.issubset(module.MODELS)) + + eager_cmd = module.build_sglang_cmd("longcat-image") + self.assertNotIn("--enable-torch-compile", eager_cmd) + self.assertIn("--enable-prompt-rewrite=false", eager_cmd) + + compiled_cmd = module.build_sglang_cmd("longcat-image", torch_compile=True) + self.assertIn("--enable-torch-compile", compiled_cmd) + + h3_cmd = module.build_sglang_cmd("minimax-h3-t2va", torch_compile=True) + self.assertNotIn("--enable-torch-compile", h3_cmd) + + def test_isolated_cache_cleanup_writes_zero_residual_ledger(self): + with tempfile.TemporaryDirectory() as tmpdir: + temp_root = Path(tmpdir) + module = _load_benchmark_module(temp_root) + cache_root = temp_root / "model-caches" + cache_dir = module._prepare_model_cache( + cache_root, "longcat-image", "baseline" + ) + + weight_path = cache_dir / "huggingface" / "hub" / "model.safetensors" + weight_path.parent.mkdir(parents=True) + weight_path.write_bytes(b"weights") + env = module._model_cache_env(cache_dir) + self.assertTrue(env["HF_HOME"].startswith(str(cache_dir))) + self.assertTrue(env["HF_XET_CACHE"].startswith(str(cache_dir))) + self.assertTrue(env["TRANSFORMERS_CACHE"].startswith(str(cache_dir))) + self.assertTrue(env["MODELSCOPE_CACHE"].startswith(str(cache_dir))) + + ledger_path = temp_root / "artifacts" / "cleanup.jsonl" + record = module._cleanup_model_cache( + cache_root, + cache_dir, + ledger_path, + "longcat-image", + "baseline", + "success", + ) + + self.assertFalse(cache_dir.exists()) + self.assertEqual(record["before"]["weight_file_count"], 1) + self.assertEqual(record["after"]["file_count"], 0) + ledger = json.loads(ledger_path.read_text(encoding="utf-8")) + self.assertEqual(ledger["exit_reason"], "success") + self.assertEqual(ledger["after"]["weight_file_count"], 0) + + def test_isolated_cache_refuses_to_reuse_existing_run_directory(self): + with tempfile.TemporaryDirectory() as tmpdir: + temp_root = Path(tmpdir) + module = _load_benchmark_module(temp_root) + cache_root = temp_root / "model-caches" + module._prepare_model_cache(cache_root, "sana-video", "baseline") + + with self.assertRaises(FileExistsError): + module._prepare_model_cache(cache_root, "sana-video", "baseline") + + def test_interrupted_run_cleans_isolated_cache_in_finally(self): + with tempfile.TemporaryDirectory() as tmpdir: + temp_root = Path(tmpdir) + module = _load_benchmark_module(temp_root) + cache_root = temp_root / "model-caches" + output_dir = temp_root / "outputs" + output_dir.mkdir() + + with ( + patch.object( + module, "_run_benchmark_once_impl", side_effect=KeyboardInterrupt + ), + self.assertRaises(KeyboardInterrupt), + ): + module.run_benchmark_once( + "sana-video", + "baseline", + output_dir, + model_cache_root=cache_root, + cleanup_model_cache=True, + ) + + self.assertFalse((cache_root / "sana-video-baseline").exists()) + ledger = json.loads( + (output_dir / "cleanup.jsonl").read_text(encoding="utf-8") + ) + self.assertEqual(ledger["exit_reason"], "interrupted") + self.assertEqual(ledger["after"]["weight_file_count"], 0) + + def test_failed_run_is_recorded_as_error_and_cleaned(self): + with tempfile.TemporaryDirectory() as tmpdir: + temp_root = Path(tmpdir) + module = _load_benchmark_module(temp_root) + cache_root = temp_root / "model-caches" + output_dir = temp_root / "outputs" + output_dir.mkdir() + + with ( + patch.object( + module, "_run_benchmark_once_impl", side_effect=RuntimeError("boom") + ), + self.assertRaisesRegex(RuntimeError, "boom"), + ): + module.run_benchmark_once( + "sana-video", + "baseline", + output_dir, + model_cache_root=cache_root, + cleanup_model_cache=True, + ) + + self.assertFalse((cache_root / "sana-video-baseline").exists()) + ledger = json.loads( + (output_dir / "cleanup.jsonl").read_text(encoding="utf-8") + ) + self.assertEqual(ledger["exit_reason"], "error") + + +if __name__ == "__main__": + unittest.main()