diff --git a/docs/cookbook/diffusion/FLUX/FLUX.mdx b/docs/cookbook/diffusion/FLUX/FLUX.mdx index 4cb84c7d88..e39331666b 100644 --- a/docs/cookbook/diffusion/FLUX/FLUX.mdx +++ b/docs/cookbook/diffusion/FLUX/FLUX.mdx @@ -64,7 +64,7 @@ CUDA_VISIBLE_DEVICES=0,1 sglang generate \ --enable-torch-compile false \ --enable-breakable-cuda-graph \ --warmup-resolutions 1024x1024 \ - --quality lossless \ + --quality exact \ --width 1024 --height 1024 \ --num-inference-steps 50 --guidance-scale 3.5 --seed 42 \ --prompt "A futuristic cyberpunk city at night, neon lights reflecting on wet streets" @@ -72,7 +72,7 @@ CUDA_VISIBLE_DEVICES=0,1 sglang generate \ Confirm `[Diffusion BCG] captured` in the log, then compare warmed request latency with eager execution on the same GPUs. Capture time and graph memory are additional startup costs. Add other served resolutions to `--warmup-resolutions`; a request with an uncaptured signature falls back to eager. -FLUX.1-dev uses a fixed 512-token T5 conditioning sequence, so changing `--bcg-text-buckets` does not create additional prompt-length graphs. Its request-gated DiT fusions at `quality=high` and `extra-high` cannot be combined with BCG: the runtime rejects those requests because the captured graph uses the lossless branches. FLUX.2 and quantized transformer overrides require separate validation. +FLUX.1-dev uses a fixed 512-token T5 conditioning sequence, so changing `--bcg-text-buckets` does not create additional prompt-length graphs. Its request-gated DiT fusions at `quality=high` and `lossless` cannot be combined with BCG: the runtime rejects those requests because the captured graph uses the lossless branches. FLUX.2 and quantized transformer overrides require separate validation. ## 4. API Usage diff --git a/docs/cookbook/diffusion/JoyEcho/JoyEcho.mdx b/docs/cookbook/diffusion/JoyEcho/JoyEcho.mdx index 05817a5dea..cdf52a4b47 100644 --- a/docs/cookbook/diffusion/JoyEcho/JoyEcho.mdx +++ b/docs/cookbook/diffusion/JoyEcho/JoyEcho.mdx @@ -206,7 +206,7 @@ CUDA_VISIBLE_DEVICES=0,1 sglang generate \ --config joy_echo_h200.json --prompt "A curious raccoon" \ --width 640 --height 384 --num-frames 33 --num-inference-steps 8 --seed 42 \ --num-gpus 2 --ulysses-degree 2 \ - --performance-mode manual --enable-torch-compile=false --quality lossless \ + --performance-mode manual --enable-torch-compile=false --quality exact \ --component-residency=all=resident --warmup-mode request \ --save-output --perf-dump-path joy_echo_h200.json.perf ``` diff --git a/docs/cookbook/diffusion/LingBot-World/LingBot-World-2.0.mdx b/docs/cookbook/diffusion/LingBot-World/LingBot-World-2.0.mdx index 384919e1ab..b81759b7b0 100644 --- a/docs/cookbook/diffusion/LingBot-World/LingBot-World-2.0.mdx +++ b/docs/cookbook/diffusion/LingBot-World/LingBot-World-2.0.mdx @@ -84,7 +84,7 @@ Send this MessagePack map immediately after the WebSocket opens. | `num_inference_steps` | integer | No | Denoising steps per chunk. LingBot defaults to `4` when omitted. | | `guidance_scale` | number | No | Classifier-free guidance scale. Realtime LingBot commonly uses `1`. | | `negative_prompt` | string | No | Negative prompt passed to the diffusion pipeline. | -| `quality` | `"lossless"`, `"extra-high"`, `"high"` | No | `lossless` keeps FP32 VAE decode. `extra-high` keeps FP32 decode and enables only any eligible request-gated kernel fusions. `high` includes those fusions and uses the validated BF16 decode path for lower per-chunk latency. | +| `quality` | `"exact"`, `"lossless"`, `"high"` | No | `lossless` keeps FP32 VAE decode. `lossless` keeps FP32 decode and enables only any eligible request-gated kernel fusions. `high` includes those fusions and uses the validated BF16 decode path for lower per-chunk latency. | | `max_chunks` | integer | No | Stop after this many chunks. Omit for a continuous session. | | `realtime_causal_sink_size` | integer | No | Number of sink frames/tokens retained in the causal attention window. | | `realtime_causal_kv_cache_num_frames` | integer | No | Number of recent frames retained in the causal KV cache window. | diff --git a/docs/cookbook/diffusion/LingBot-World/LingBot-World.mdx b/docs/cookbook/diffusion/LingBot-World/LingBot-World.mdx index a3dea53259..1f3e47d55b 100644 --- a/docs/cookbook/diffusion/LingBot-World/LingBot-World.mdx +++ b/docs/cookbook/diffusion/LingBot-World/LingBot-World.mdx @@ -79,7 +79,7 @@ Send this MessagePack map immediately after the WebSocket opens. | `num_inference_steps` | integer | No | Denoising steps per chunk. LingBot defaults to `4` when omitted. | | `guidance_scale` | number | No | Classifier-free guidance scale. Realtime LingBot commonly uses `1`. | | `negative_prompt` | string | No | Negative prompt passed to the diffusion pipeline. | -| `quality` | `"lossless"`, `"extra-high"`, `"high"` | No | `lossless` keeps FP32 VAE decode. `extra-high` keeps FP32 decode and enables only any eligible request-gated kernel fusions. `high` includes those fusions and uses the validated BF16 decode path for lower per-chunk latency. | +| `quality` | `"exact"`, `"lossless"`, `"high"` | No | `lossless` keeps FP32 VAE decode. `lossless` keeps FP32 decode and enables only any eligible request-gated kernel fusions. `high` includes those fusions and uses the validated BF16 decode path for lower per-chunk latency. | | `max_chunks` | integer | No | Stop after this many chunks. Omit for a continuous session. | | `realtime_causal_sink_size` | integer | No | Number of sink frames/tokens retained in the causal attention window. | | `realtime_causal_kv_cache_num_frames` | integer | No | Number of recent frames retained in the causal KV cache window. | diff --git a/docs/cookbook/diffusion/MiniMax/MiniMax-H3.mdx b/docs/cookbook/diffusion/MiniMax/MiniMax-H3.mdx index 87f8e86406..46fac1627a 100644 --- a/docs/cookbook/diffusion/MiniMax/MiniMax-H3.mdx +++ b/docs/cookbook/diffusion/MiniMax/MiniMax-H3.mdx @@ -95,7 +95,7 @@ for the baseline; the B200/B300 quantization measurements and H200 Cache-DiT quality audit do not establish coverage on Grace Blackwell. GB300 end-to-end coverage currently includes single-host FL2VA text-to-video -with audio, native precision, eager execution, and `quality="lossless"`. +with audio, native precision, eager execution, and `quality="exact"`. Ref2VA and multi-host commands remain **Unverified**. See the [GB300 measurements](#gb300-single-host) for the tested workload. @@ -348,7 +348,7 @@ their [usual meanings](#4-generate-video-and-audio), including `conditions: []` for `t2va` and keyframes for `fl2va`. Start with `quality: "lossless"` and the standard 50-point schedule. Here -`lossless` disables additional request-time approximations; it does not undo +`exact` disables additional request-time approximations; it does not undo INT8 quantization, pruning, or fine-tuning. The author's recommended Ref2V Turbo LoRA is a separate optional adapter, not an automatically inferred 4-step schedule. Reference fidelity can differ from the official model; see @@ -1008,18 +1008,18 @@ done `quality` is a cumulative request-scoped optimization parameter with three levels: -- `"lossless"` (default): the exact reference path. Output is bit-exact +- `"exact"` (default): the exact reference path. Output is bit-exact against the reference implementation and the CI ground truth. -- `"extra-high"`: includes the global fusion-only tier but does not enable +- `"lossless"`: includes the global fusion-only tier but does not enable Cache-DiT or another approximate optimization. MiniMax-H3 currently has no - request-gated fusion site, so its denoise path is the same as `lossless`. + request-gated fusion site, so its denoise path is the same as `exact`. - `"high"`: the audited accelerated path. Quality is guaranteed (the audited Cache-DiT configuration measures SSIM 0.931 / PSNR 28.16 dB against `lossless`), but output is no longer bit-identical to the reference. One resident server serves all three levels; a `quality: "high"` request mounts its audited Cache-DiT policy at the batch boundary, and a later -`quality: "lossless"` or `quality: "extra-high"` request removes the hooks +`quality: "lossless"` or `quality: "lossless"` request removes the hooks before denoising. Start the validated server once: @@ -1057,14 +1057,14 @@ omitting the field is equivalent. - + The global fusion-only tier. It does not enable MiniMax-H3 Cache-DiT and -currently follows the same H3 denoise path as `lossless`. +currently follows the same H3 denoise path as `exact`. ```json Request field { - "quality": "extra-high" + "quality": "lossless" } ``` @@ -1089,8 +1089,8 @@ The measured trade-off is: | `quality` | Mean
inference
latency | Speedup | SSIM vs
lossless | PSNR vs
lossless | Expected
trade-off | | --- | ---: | ---: | ---: | ---: | --- | -| `lossless` | 75.10 s | 1.00× | 1.000 | exact | Native reference path | -| `extra-high` | Not separately measured | — | Same H3 denoise path | Same H3 denoise path | Fusion-only tier; no H3-specific request-gated site yet | +| `exact` | 75.10 s | 1.00× | 1.000 | exact | Native reference path | +| `lossless` | Not separately measured | — | Same H3 denoise path | Same H3 denoise path | Fusion-only tier; no H3-specific request-gated site yet | | `high` | 53.70 s | 1.40× | 0.931 | 28.16 dB | Smallest same-seed visual change | These numbers use 1344×768, 124-frame, 24 fps T2VA with 50 inference steps, @@ -1099,7 +1099,7 @@ video flow shift 12, audio flow shift 3, and three fixed prompt/seed pairs on and a moving close-up portrait. `inference_time_s` is averaged across the three prompts; the quiet-scene point is itself the mean of two repeats. -SSIM and PSNR compare decoded, frame-aligned output with the `lossless` +SSIM and PSNR compare decoded, frame-aligned output with the `exact` result for the same prompt and seed. They measure trajectory deviation, not absolute perceptual quality: the `high` path can produce a different but still plausible realization. It also changes the joint audio-video denoise @@ -1118,7 +1118,7 @@ name, for example `sglang generate --quality high`. For manually tuned Cache-DiT experiments outside that validated path, omit the request `quality` field and set `--enable-cache-dit` or the process-wide `SGLANG_CACHE_DIT_*` defaults. Any explicit `quality`, including -`"lossless"` and `"extra-high"`, takes H3 off the generic Cache-DiT path. The +`"exact"` and `"lossless"`, takes H3 off the generic Cache-DiT path. The 24 GB layerwise recipe above can use the same switch; skipped blocks are not streamed. @@ -1503,7 +1503,7 @@ Dynamic cuDNN/FA attention, and eager execution. The model was downloaded by `sglang serve` from `MiniMaxAI/MiniMax-H3`. The workload uses the builder's cat-band prompt, FL2VA T2VA, a 5-second request -at a 768-pixel short edge, 50 inference steps, `quality="lossless"`, one output, +at a 768-pixel short edge, 50 inference steps, `quality="exact"`, one output, and seed 1101. Every MP4 contained video and audio and passed a full decode check. | First full request after server warmup | Following three requests | Warm median | diff --git a/docs/cookbook/diffusion/Qwen-Image/Qwen-Image-Edit.mdx b/docs/cookbook/diffusion/Qwen-Image/Qwen-Image-Edit.mdx index 5b7c92e1e3..5b5c585130 100644 --- a/docs/cookbook/diffusion/Qwen-Image/Qwen-Image-Edit.mdx +++ b/docs/cookbook/diffusion/Qwen-Image/Qwen-Image-Edit.mdx @@ -60,7 +60,7 @@ CUDA_VISIBLE_DEVICES=0,1 sglang generate \ --cfg-parallel-size 2 \ --tp-size 1 \ --ulysses-degree 1 \ - --quality lossless \ + --quality exact \ --enable-torch-compile false \ --warmup-mode request \ --width 640 --height 640 --num-frames 4 \ diff --git a/docs/docs/sglang-diffusion/api/cli.mdx b/docs/docs/sglang-diffusion/api/cli.mdx index b547bdb395..bcb363ab20 100644 --- a/docs/docs/sglang-diffusion/api/cli.mdx +++ b/docs/docs/sglang-diffusion/api/cli.mdx @@ -100,7 +100,7 @@ Use `sglang generate --help` and `sglang serve --help` for the full argument lis - `--warmup-mode {off|request|server}`: control startup warmup for `sglang serve`; `off` skips warmup, `request` primes the request path, and `server` runs a full synthetic server warmup before serving traffic - `--enable-torch-compile {true|false}`: compile native diffusion hot paths. When no warmup mode is configured, this also enables server warmup so first real requests do not pay compile latency. - `--offload-during-compile {true|false}`: when compile warmup is active, temporarily layerwise-offload DiT weights and move resident non-DiT components off-device so `max-autotune` fits on tighter-memory GPUs; the configured serving residency is restored before real traffic. Skipped under existing layerwise offload, Cache-DiT, or FSDP. -- `--enable-breakable-cuda-graph {true|false}`: capture supported DiT forwards as breakable CUDA graph segments to reduce launch overhead. Requires `--warmup-resolutions` for every served resolution because each resolution is captured separately. An `extra-high` or `high` request is rejected when it would mount request-scoped DiT fusions that were not present during lossless graph capture; VAE-only request-gated paths remain compatible. +- `--enable-breakable-cuda-graph {true|false}`: capture supported DiT forwards as breakable CUDA graph segments to reduce launch overhead. Requires `--warmup-resolutions` for every served resolution because each resolution is captured separately. An `lossless` or `high` request is rejected when it would mount request-scoped DiT fusions that were not present during lossless graph capture; VAE-only request-gated paths remain compatible. - `--bcg-text-buckets {N...}`: prompt-length padding buckets for breakable CUDA graph capture/replay reuse. - `--attention-backend {BACKEND}`: attention backend for native SGLang and diffusers pipelines - `--enable-attention-backend-autotune {true|false}`: for SGLang-native pipelines, benchmark compatible attention backends on each layer's first sufficiently large input and keep a backend only when it is both numerically compatible and measurably faster. Disabled by default and currently validated on SM90 and SM12x. Explicit `--attention-backend`, component overrides, and model-required backends are never replaced. @@ -123,7 +123,7 @@ Use `sglang generate --help` and `sglang serve --help` for the full argument lis - `--image-path {PATH} [{PATH} ...]`: input image(s) for image-to-video or image-to-image generation - `--num-inference-steps {STEPS}` and `--seed {SEED}` - `--num-outputs-per-prompt {N}` / `--num-outputs {N}`: generate multiple outputs for each prompt. A scalar seed expands as `seed + output_index`. -- `--quality {lossless,extra-high,high}`: cumulative request-level optimization tier. `lossless` (default) keeps the selected deployment's reference path and all unconditional bit-exact replacements. `extra-high` adds only request-gated DiT/VAE kernel fusions; the tier does not itself enable sparse, caching, or other approximate paths. `high` includes the complete `extra-high` set and may also enable model-owned approximate optimizations. Separately configured quantization, attention, or caching options still apply. Support and validation constraints are model-specific. +- `--quality {exact,lossless,high}`: cumulative request-level optimization tier. `lossless` (default) runs every fast path that keeps the reference math and the precision of every operand, moving only where the rounding happens; it does not enable quantization, sparsity, caching, or another approximate path. `exact` restricts it to the fast paths that reproduce the reference's rounding exactly, so output is bit-identical in the same environment -- ask for it when a run has to be reproducible. `high` includes the complete `lossless` set and may also enable model-owned approximate optimizations. `extra-high` is accepted as the former name of `lossless`. Separately configured quantization, attention, or caching options still apply. Support and validation constraints are model-specific. - `skip_softmax_params` (online request only): explicitly enables lossy BLASST/Skip-Softmax attention for compatible FA self-attention layers. See [Attention Backends](../attention_backends#request-scoped-skip-softmax). - `--height {HEIGHT}`, `--width {WIDTH}`, `--num-frames {N}`, `--fps {FPS}` - `--output-path {PATH}`, `--output-file-name {NAME}`, `--save-output`, `--return-frames` diff --git a/docs/docs/sglang-diffusion/api/openai_api.mdx b/docs/docs/sglang-diffusion/api/openai_api.mdx index ea46cdffe0..9d0d2e82b9 100644 --- a/docs/docs/sglang-diffusion/api/openai_api.mdx +++ b/docs/docs/sglang-diffusion/api/openai_api.mdx @@ -169,7 +169,7 @@ The server implements an OpenAI-compatible Images API under the `/v1/images` nam #### Request quality -`quality` selects a cumulative request-level optimization tier: `lossless` keeps the selected deployment's reference path and all unconditional bit-exact replacements; `extra-high` additionally enables only request-gated DiT/VAE kernel fusions; `high` includes the full `extra-high` set and may also enable model-owned sparse, caching, lower-precision, or other approximate paths. The tier does not override separately configured quantization, attention, or caching options. Omit it (or send OpenAI's default `auto`) to keep the `lossless` runtime default. It is distinct from `output_quality`, which controls only output-file compression. The same extension is accepted by image edits and video requests. +`quality` selects a cumulative request-level optimization tier: `exact` keeps the selected deployment's reference path and all unconditional bit-exact replacements, so output is bit-identical to it in the same environment; `lossless` additionally enables the request-gated DiT/VAE fusions that keep the reference math and every operand's precision and only move the rounding; `high` includes the full `lossless` set and may also enable model-owned sparse, caching, lower-precision, or other approximate paths. `extra-high` is accepted as the former name of `lossless`. The tier does not override separately configured quantization, attention, or caching options. Omit it (or send OpenAI's default `auto`) to keep the `lossless` runtime default. It is distinct from `output_quality`, which controls only output-file compression. The same extension is accepted by image edits and video requests. **Python Example (b64_json response):** diff --git a/docs/docs/sglang-diffusion/fused_kernels.mdx b/docs/docs/sglang-diffusion/fused_kernels.mdx index 138ed98ded..a96f028cdf 100644 --- a/docs/docs/sglang-diffusion/fused_kernels.mdx +++ b/docs/docs/sglang-diffusion/fused_kernels.mdx @@ -14,7 +14,9 @@ Multi-step denoising amplifies a per-step rounding difference into visible quali **Bit-exact — mounted unconditionally.** The kernel reproduces every rounding boundary of the eager chain, so `torch.equal` holds against the reference. Some go quite far to get there: the fused LayerNorm+modulate kernel replicates PyTorch's `vectorized_layer_norm_kernel` down to its Welford update order, guarded reciprocal, and warp-fold tree; the fused RMSNorm+scale/shift kernel replicates FlashInfer's CuTe-DSL `RMSNormKernel` fragment order and `shfl.bfly` fold. Because the dispatch they replicate can change underneath them, each one still verifies itself against the live eager chain on first sight and falls back permanently on any mismatch. -**Not bit-exact — request-gated.** These differ from eager only at half-precision rounding-order level, but that is enough to matter, so they are mounted only for `quality="extra-high"` and `quality="high"` requests, at batch boundaries, all-or-nothing per transformer. The default `quality="lossless"` runs the unmodified reference chain. +**Same math, different rounding — request-gated at `lossless`.** These keep the reference function and the precision of every operand and accumulator; only the place or the order of the rounding moves, so the result is within rounding error of the reference rather than equal to it. They are mounted for `quality="lossless"` (the default) and `quality="high"` requests, at batch boundaries, all-or-nothing per transformer. `quality="exact"` runs the unmodified reference chain instead. + +**Approximate — request-gated at `high`.** A fusion that quantizes, lowers an operand or accumulator precision, or sits upstream of a quantizer or threshold is no longer a rounding difference, so it needs model-level quality evaluation and is mounted only for `quality="high"`. Today that is the FLUX.2 NVFP4 FC1+SwiGLU+quant path (it quantizes FC2's input before the reference BF16 intermediate exists), Ideogram's BF16-native gate RMSNorm (the reference keeps the norm statistics in fp32), and SANA-Video's BF16-input linear attention (the reference promotes those inputs to FP32). **Model/checkpoint-native.** Generic close-contract kernels, sparse operators, and FP8/NVFP4 producers can be part of a model implementation or a separately selected deployment path. They are documented in the inventory, but `quality` does not select or undo those choices. @@ -28,11 +30,18 @@ The quality levels are cumulative: | `quality` | Included optimization set | | --- | --- | -| `lossless` | The selected deployment's reference path plus every unconditional bit-exact replacement | -| `extra-high` | Everything in `lossless`, plus request-gated DiT and VAE kernel fusions; this level does not itself enable sparse, caching, or another approximate path | -| `high` | Everything in `extra-high`, plus any model-owned high-only optimization, such as an audited Cache-DiT policy or lower-precision VAE decode | +| `exact` | The selected deployment's reference path plus every unconditional bit-exact replacement | +| `lossless` | Everything in `exact`, plus request-gated DiT and VAE fusions that keep the reference math and every operand's precision; this level does not itself enable quantization, sparsity, caching, or another approximate path | +| `high` | Everything in `lossless`, plus any model-owned approximate optimization, such as an audited Cache-DiT policy, lower-precision VAE decode, or the approximate fusions listed above | -If a model has no eligible request-gated fusion, `extra-high` can execute the same path as `lossless`. Likewise, `high` adds only the model-specific high-only paths that the active pipeline implements. +A fusion claiming `lossless` has to pass the admission checks in +`sglang/multimodal_gen/test/quality_tier_admission.py`: scored against an fp64 +evaluation of the same math, its RMS and max error may not exceed 1.5x the +path the model runs today, its rounding may not be biased, and NaN, Inf and +non-contiguous inputs have to behave as before. Lowering an accumulation below +fp32, or quantizing, makes it `high` by construction — no measurement admits it. + +If a model has no eligible request-gated fusion, `lossless` can execute the same path as `exact`. Likewise, `high` adds only the model-specific high-only paths that the active pipeline implements. `quality` is not a master precision switch. A quantized checkpoint, an explicitly selected approximate attention backend, or an independently enabled cache remains active at every quality tier. @@ -41,15 +50,15 @@ If a model has no eligible request-gated fusion, `extra-high` can execute the sa ## Enabling the request-gated set ```bash -sglang generate --model-path MODEL_PATH --prompt "..." --quality extra-high +sglang generate --model-path MODEL_PATH --prompt "..." --quality lossless ``` -The server default stays `lossless`; the OpenAI-compatible endpoints carry it per request. Images: +The server default is `lossless`, so this is the set a request gets unless it asks otherwise; the OpenAI-compatible endpoints carry the level per request. Images: ```bash curl -X POST http://${HOST}:${PORT}/v1/images/generations \ -H 'Content-Type: application/json' \ - -d '{"model": "MODEL_PATH", "prompt": "...", "quality": "extra-high"}' + -d '{"model": "MODEL_PATH", "prompt": "...", "quality": "lossless"}' ``` Video, same field: @@ -57,7 +66,7 @@ Video, same field: ```bash curl -X POST http://${HOST}:${PORT}/v1/videos \ -H 'Content-Type: application/json' \ - -d '{"model": "MODEL_PATH", "prompt": "...", "quality": "extra-high"}' + -d '{"model": "MODEL_PATH", "prompt": "...", "quality": "lossless"}' ``` @@ -68,14 +77,14 @@ The `quality` field in a **video response** body is unrelated. It is Sora-compat Do not combine request-gated DiT fusions with `--enable-breakable-cuda-graph`. -BCG warmup captures the lossless module branches before an `extra-high` or +BCG warmup captures the lossless module branches before an `lossless` or `high` request mounts its DiT fusions, so replay would bypass the requested kernels. SGLang rejects this combination for models with eligible DiT quality sites. Models whose request-gated path changes only VAE decode remain allowed because BCG captures the DiT only. -These fusion families mount under both `quality="extra-high"` and +These fusion families mount under both `quality="lossless"` and `quality="high"`: | Fusion | What it folds | @@ -237,11 +246,11 @@ Kernels are written against a specific eager chain in a specific model, so cover | LTX-2 | QK-norm + split RoPE, ada-values split, RMSNorm+modulate, modulate, residual-gate add, linear+GELU | | LTX-2.5 decoder | paired 3D RoPE with shared axis-table cache | | HunyuanVideo / Helios | QKV+RoPE pack, strided QK RMSNorm, linear+GELU; Helios also has paired in-place Q/K RoPE | -| LingBot Video MoE | Default-on group-limited top-k expert selection; fused RMSNorm, per-token gated residual, and fused RMSNorm+modulate at `quality=extra-high` or `quality=high` | +| LingBot Video MoE | Default-on group-limited top-k expert selection; fused RMSNorm, per-token gated residual, and fused RMSNorm+modulate at `quality=lossless` or `quality=high` | | Sana | LN+modulate, GLUMB bias+SiLU / bias+GLU, residual-gate add | -| SANA-Video | Packed QKV/KV; paired fp64 interleaved RoPE; GLUMB bias+SiLU / bias+GLU in eager and BCG; LN+modulate and residual-gate add during BCG; BF16-input linear attention at `quality=extra-high` or `quality=high` | +| SANA-Video | Packed QKV/KV; paired fp64 interleaved RoPE; GLUMB bias+SiLU / bias+GLU in eager and BCG; LN+modulate and residual-gate add during BCG; BF16-input linear attention at `quality=lossless` or `quality=high` | | Sana-WM | bidirectional gated delta-net, fused QK inverse-RMS | -| Wan / LongLive 2 VAE | temb table slices (Wan); VAE cat+pad and DupUp3D add; lossless FP32 normalization post-ops; full `channels_last_3d` RMSNorm+SiLU at `quality=extra-high` or `quality=high` | +| Wan / LongLive 2 VAE | temb table slices (Wan); VAE cat+pad and DupUp3D add; lossless FP32 normalization post-ops; full `channels_last_3d` RMSNorm+SiLU at `quality=lossless` or `quality=high` | | Cosmos3 | QK-norm + RoPE + KV packing; rounded T2I fusion also supports Hopper TP2/SP1 Super-Text2Image with the 5120-wide SwiGLU architecture | | Krea2 / MiniMax-H3 | QK-norm + RoPE (Krea2 also CuTe-DSL norm+scale/shift; MiniMax-H3 also indexed modulation) | | FLUX.2 VAE / HunyuanVAE / latent upsampler | GroupNorm + SiLU (channels-last two-pass for FLUX.2) | diff --git a/docs/docs/sglang-diffusion/performance-optimization.mdx b/docs/docs/sglang-diffusion/performance-optimization.mdx index d8e04247e8..6ed9f386ac 100644 --- a/docs/docs/sglang-diffusion/performance-optimization.mdx +++ b/docs/docs/sglang-diffusion/performance-optimization.mdx @@ -25,16 +25,53 @@ stricter tier. | Tier | Optimization boundary | | --- | --- | -| `lossless` (default) | The selected deployment's reference execution plus all unconditional bit-exact replacements | -| `extra-high` | Everything in `lossless`, plus only request-gated DiT/VAE kernel fusions; the tier does not itself enable sparse, caching, or other approximate paths | -| `high` | Everything in `extra-high`, plus model-owned high-only paths such as an audited Cache-DiT policy or lower-precision VAE decode | +| `exact` | The selected deployment's reference execution plus all unconditional bit-exact replacements. Output is bit-identical to the reference path in the same environment | +| `lossless` (default) | Everything in `exact`, plus request-gated DiT/VAE fusions that keep the reference math and the precision of every operand and accumulator, changing only where or in which order the rounding happens. Output matches the reference to rounding error, not bit for bit | +| `high` | Everything in `lossless`, plus model-owned paths that change what is computed -- lower precision, sparsity, or reuse across steps, such as an audited Cache-DiT policy or lower-precision VAE decode | -Use `extra-high` when you want to isolate fusion wins from approximate -acceleration. A tier may be a no-op when the active model has no eligible path. +Bit-identical describes reproducibility, not quality: a `lossless` fast path is +the same mathematical function, and its rounding is no further from the exact +value than the reference's own. On FLUX.1-dev the two tiers differ by 34.5 dB +PSNR, while the same tier differs by 32.5 dB between one GPU and two -- the +tier moves the image less than the parallel layout does, which is why it is +the default. Ask for `exact` when a run has to be byte-for-byte reproducible, +or when bisecting a numerical bug. + +`extra-high` is accepted as the former name of `lossless`. Note that +`lossless` itself was the former name of `exact`, and that it is now the +default: a request that passed `quality="lossless"` for bit-identical output +has to pass `quality="exact"`, and a deployment that never set `quality` moves +from the bit-exact path to this one. + +A tier may be a no-op when the active model has no eligible path. Separately configured quantization, attention, or caching options still apply. See [Fused Kernels](/docs/sglang-diffusion/fused_kernels) for the current request-gated families and their numerical contracts. +### Library numerics below the tiers + +PyTorch enables two approximations by default, underneath the whole tier +system: `torch.backends.cudnn.allow_tf32` truncates fp32 convolution inputs to +10 mantissa bits, and +`torch.backends.cuda.matmul.allow_bf16_reduced_precision_reduction` lets a bf16 +GEMM accumulate split-K partials below fp32. Both change what is computed, so +by the definitions above they belong to `high`. + +They are process-global and a server mixes tiers across concurrent batches, so +they cannot follow a request. The runtime sets both explicitly per worker and +logs the result, instead of inheriting whatever the installed PyTorch defaults +to: + +| Flag | Default | Effect of turning it off | +| --- | --- | --- | +| `--allow-cudnn-tf32` | `true` (PyTorch's own default) | fp32 convolutions run on fp32 CUDA cores instead of TF32 tensor cores. Affects the VAE decoders that default to `vae_precision="fp32"` | +| `--allow-bf16-reduced-precision-reduction` | `true` (PyTorch's own default) | bf16 GEMMs accumulate split-K partials at fp32 | + +The defaults keep PyTorch's values so that `exact` is no slower than it was +before these flags existed. A deployment that needs the strict contract — fp32 +and bf16 math at reference precision throughout — passes both as `false` and +accepts the slower convolutions. + ## Output-Preserving / Lossless-Style Levers These settings should preserve model behavior while changing residency, parallelism, kernels, or scheduling. They are the first choices for production tuning. @@ -199,7 +236,7 @@ These techniques can change the denoising path, numerical representation, or gen 1. Establish a baseline with the target model, resolution, frame count, step count, and GPU type. 2. Select `--performance-mode` and explicit residency or parallelism flags. -3. Compare `quality=lossless` with `quality=extra-high` to isolate the request-gated fusion set. +3. Compare `quality=lossless` with `quality=lossless` to isolate the request-gated fusion set. 4. Compare breakable CUDA graph against eager execution for supported fixed-shape pipelines. Pass every served resolution to `--warmup-resolutions` and confirm capture in the server log. Models with request-gated DiT fusions cannot combine those fusions with a graph captured from the lossless branches. 5. Tune attention backend and batching for the deployment pattern. 6. Profile if the bottleneck is unclear. diff --git a/docs/scripts/check_cookbook_configs.mjs b/docs/scripts/check_cookbook_configs.mjs index 3e4b4fa6dc..db338a45e1 100755 --- a/docs/scripts/check_cookbook_configs.mjs +++ b/docs/scripts/check_cookbook_configs.mjs @@ -174,6 +174,9 @@ for (const path of walk(CONFIGS)) { for (const dim of (config.overlayDims || [])) { const ids = (dim.options || []).map((o) => o.id); + if (new Set(ids).size !== ids.length) { + fail(where, `overlayDims.${dim.id} has duplicate option ids`); + } if (dim.kind === "number") { if (!Number.isInteger(dim.min) || !Number.isInteger(dim.max) || dim.min > dim.max) { fail(where, `overlayDims.${dim.id} has invalid numeric bounds`); @@ -316,7 +319,7 @@ for (const path of walk(CONFIGS)) { { hw: "gb200" }, { hw: "gb300", mode: "i2va" }, { hw: "gb300", weights: "ref2va", mode: "v2v" }, - { hw: "gb300", quality: "extra-high" }, + { hw: "gb300", quality: "lossless" }, { hw: "gb300", outputs: "2" }, { hw: "gb300", precision: "fp8" }, { hw: "gb300", attention: "sage" }, diff --git a/docs/src/snippets/configs/MiniMaxAI/minimax-h3.jsx b/docs/src/snippets/configs/MiniMaxAI/minimax-h3.jsx index d719abecfa..95a56a605e 100644 --- a/docs/src/snippets/configs/MiniMaxAI/minimax-h3.jsx +++ b/docs/src/snippets/configs/MiniMaxAI/minimax-h3.jsx @@ -527,18 +527,18 @@ return { description: "Cumulative reference, fusion-only, or audited Cache-DiT execution.", quality: "Sampling policy", learnMore: "#choose-the-quality-level", - default: "lossless", + default: "exact", options: [ { - id: "lossless", - label: "Lossless", + id: "exact", + label: "Exact", recommended: true, description: "Reference-exact denoising without Cache-DiT approximation.", }, { - id: "extra-high", - label: "Extra high", - description: "Includes fusion-only request paths but not Cache-DiT; MiniMax-H3 currently follows its lossless denoise path at this tier.", + id: "lossless", + label: "Lossless", + description: "Includes fusion-only request paths but not Cache-DiT; MiniMax-H3 currently follows its exact denoise path at this tier.", }, { id: "high", @@ -716,8 +716,8 @@ return { const serveVerified = topologyVerified && encoderVerified && attentionVerified && precisionVerified && executionVerified && checkpointVerified; const requestCovered = s.hw !== "gb300" || (serveVerified && s.weights === "fl2va" - && s.mode === "t2va" && s.quality === "lossless" && Number(s.outputs) === 1); - const requestVerified = topologyVerified && requestCovered && (["lossless", "extra-high"].includes(s.quality) + && s.mode === "t2va" && s.quality === "exact" && Number(s.outputs) === 1); + const requestVerified = topologyVerified && requestCovered && (["exact", "lossless"].includes(s.quality) || (s.quality === "high" && highAudited && s.execution === "eager")); const topologyParts = []; diff --git a/python/sglang/kernels/ops/diffusion/README.md b/python/sglang/kernels/ops/diffusion/README.md index ca656afefe..976993c3eb 100644 --- a/python/sglang/kernels/ops/diffusion/README.md +++ b/python/sglang/kernels/ops/diffusion/README.md @@ -60,11 +60,12 @@ themselves against the live eager chain on first sight via `sites/bitexact_gate.py` and fall back permanently on mismatch — the dispatch they replicate can change under them. -**Not bit-exact → quality-gated.** Mounted onto marked `nn.Module` sites only -for `quality="extra-high"` and `quality="high"` requests, at batch boundaries, -all-or-nothing per transformer (`sites/quality_gate.py`). `extra-high` adds -only these request-gated DiT/VAE fusions; `high` is cumulative and may also -enable model-owned approximate paths such as Cache-DiT or a lower-precision +**Not bit-exact → quality-gated.** Mounted onto marked `nn.Module` sites at +batch boundaries, all-or-nothing per transformer (`sites/quality_gate.py`), +from the lowest tier that may run each one: `lossless` for a fusion that keeps +the reference math and every operand's precision and only moves the rounding, +`high` for one that quantizes or lowers a precision. `high` is cumulative and +may also enable model-owned approximate paths such as Cache-DiT or a lower-precision decode. A plain fp32 single-pass norm fusion looks harmless and is not: on ERNIE-Image it moved the 50-step trajectory to PSNR 18.83 dB, which is what motivated the bit-exact rewrite. @@ -248,7 +249,11 @@ inspecting model modules is its whole job. wrapper when the fallback policy is shared. 4. State the numerical contract in the module docstring, including which shapes it was verified on. -5. If it is not bit-exact, gate it through `sites/`. It must mount for both - `extra-high` and `high`, never for the default `lossless` path. +5. If it is not bit-exact, gate it through `sites/` and declare its tier in + `_QUALITY_FUSION_HANDLERS`: `lossless` when it keeps the reference math and + every operand's precision, `high` when it does not. Never `exact`, which + is reserved for paths that reproduce the reference's rounding. A + `lossless` claim has to pass + `multimodal_gen/test/quality_tier_admission.py`. 6. Test it in the domain suite (`test/registered/kernels/ops/diffusion/`), and the model wiring in `test_model_fast_paths.py`. diff --git a/python/sglang/kernels/ops/diffusion/__init__.py b/python/sglang/kernels/ops/diffusion/__init__.py index d1d2a869dc..dd5569d5db 100644 --- a/python/sglang/kernels/ops/diffusion/__init__.py +++ b/python/sglang/kernels/ops/diffusion/__init__.py @@ -642,6 +642,7 @@ _EXPORTS: dict[str, str] = { "can_use_flux2_strided_qknorm_rope": "rope.flux2_qknorm_rope_triton", "flux2_strided_qknorm_rope": "rope.flux2_qknorm_rope_triton", "rmsnorm_preserve_reduction": "norm.rmsnorm_preserve_reduction", + "can_use_fused_rmsnorm_modulation": "norm.rmsnorm_scale_shift_bitexact", "can_use_fused_rmsnorm_scale_shift": "norm.rmsnorm_scale_shift_bitexact", "fused_rmsnorm_scale_shift_bitexact": "norm.rmsnorm_scale_shift_bitexact", "fused_scale_residual_rmsnorm_scale_shift_bitexact": "norm.rmsnorm_scale_shift_bitexact", diff --git a/python/sglang/kernels/ops/diffusion/norm/rmsnorm_scale_shift_bitexact.py b/python/sglang/kernels/ops/diffusion/norm/rmsnorm_scale_shift_bitexact.py index 7b85c65183..b6082dda39 100644 --- a/python/sglang/kernels/ops/diffusion/norm/rmsnorm_scale_shift_bitexact.py +++ b/python/sglang/kernels/ops/diffusion/norm/rmsnorm_scale_shift_bitexact.py @@ -164,6 +164,32 @@ def _is_row_broadcast(t: torch.Tensor, shape: tuple, device: torch.device) -> bo ) +def can_use_fused_rmsnorm_modulation( + x: torch.Tensor, scale: torch.Tensor, shift: torch.Tensor +) -> bool: + """Whether this kernel can serve ``x`` with this ``scale``/``shift``. + + The launcher raises on everything it cannot serve, which is the right + answer for a caller that has already committed. A request-gated caller has + not: a fast path that meets an input it cannot handle has to fall back, + not fail the request. So it asks here first, and this mirrors exactly what + :func:`_validate_rmsnorm_modulate` checks about ``x`` and the modulation + rows -- a dtype-and-width check alone leaves the layout of ``scale`` and + ``shift`` to chance. + """ + if not ( + can_use_fused_rmsnorm_scale_shift(x.dtype, x.shape[-1]) + and x.is_cuda + and x.ndim == 3 + and x.is_contiguous() + ): + return False + row_shape = (x.shape[0], 1, x.shape[-1]) + return _is_row_broadcast(scale, row_shape, x.device) and _is_row_broadcast( + shift, row_shape, x.device + ) + + def can_use_fused_rmsnorm_scale_shift(dtype: torch.dtype, hidden: int) -> bool: """Select the BF16 FlashInfer reduction replicated by this kernel.""" return ( diff --git a/python/sglang/kernels/ops/diffusion/norm/rmsnorm_scale_shift_triton.py b/python/sglang/kernels/ops/diffusion/norm/rmsnorm_scale_shift_triton.py index cd1c745adc..89de03a9b9 100644 --- a/python/sglang/kernels/ops/diffusion/norm/rmsnorm_scale_shift_triton.py +++ b/python/sglang/kernels/ops/diffusion/norm/rmsnorm_scale_shift_triton.py @@ -15,7 +15,7 @@ eager LingBot chain ``(norm(x) * (1 + scale) + shift).to(dtype)``. Unlike the bit-exact ``rmsnorm_scale_shift_bitexact`` (ERNIE), this kernel does *not* reproduce PyTorch's parallel variance reduction order bit-for-bit, so it -is intended for the request-gated (``quality="extra-high"``/``"high"``) fusion +is intended for the request-gated (``quality="lossless"``/``"high"``) fusion path, matching the existing quality-gated LingBot RMSNorm fusion. """ diff --git a/python/sglang/kernels/ops/diffusion/sites/bitexact_gate.py b/python/sglang/kernels/ops/diffusion/sites/bitexact_gate.py index a7be00ec95..9eb8ae182d 100644 --- a/python/sglang/kernels/ops/diffusion/sites/bitexact_gate.py +++ b/python/sglang/kernels/ops/diffusion/sites/bitexact_gate.py @@ -63,6 +63,13 @@ def flashinfer_rmsnorm_diagnostic_hint() -> str: class BitExactFusionGate: """Track permanent disable + first-sight ``torch.equal`` verification. + This is the ``exact`` tier's mechanism: it backs a fast path that is + bit-exact by construction, so the path mounts unconditionally and the gate + only catches the case where the dispatch it replicates changed underneath + it. A fast path whose rounding differs from the reference belongs to a + later tier and uses ``QualityGatedFusion`` instead, which mounts per + request from the tier declared in ``_QUALITY_FUSION_HANDLERS``. + Two modes: * **once-for-all** (default): the first successful equal-check enables the diff --git a/python/sglang/kernels/ops/diffusion/sites/flux2_nvfp4_swiglu_quant_site.py b/python/sglang/kernels/ops/diffusion/sites/flux2_nvfp4_swiglu_quant_site.py index ae73cca3ab..03adeb0bdb 100644 --- a/python/sglang/kernels/ops/diffusion/sites/flux2_nvfp4_swiglu_quant_site.py +++ b/python/sglang/kernels/ops/diffusion/sites/flux2_nvfp4_swiglu_quant_site.py @@ -2,7 +2,7 @@ The fused FC1 + SwiGLU + FC2-input quantization path changes the rounding order by quantizing before the reference BF16 intermediate is materialized. -Keep it disabled for the lossless default and mount it only for +Keep it disabled below the approximate tier and mount it only for ``quality="high"`` requests at denoising batch boundaries. """ diff --git a/python/sglang/kernels/ops/diffusion/sites/fused_gate_rmsnorm_site.py b/python/sglang/kernels/ops/diffusion/sites/fused_gate_rmsnorm_site.py index 46aae1844b..6e5c73a896 100644 --- a/python/sglang/kernels/ops/diffusion/sites/fused_gate_rmsnorm_site.py +++ b/python/sglang/kernels/ops/diffusion/sites/fused_gate_rmsnorm_site.py @@ -14,7 +14,7 @@ statistics round through bf16) but **not bit-exact**. Following the fused-linear-GELU precedent, sites are therefore mounted only for ``quality="high"`` requests via :func:`mount_fused_gate_rmsnorm` / :func:`unmount_fused_gate_rmsnorm` at batch boundaries; the default -``"lossless"`` path keeps the unmodified reference chain bit-for-bit. +``"exact"`` path keeps the unmodified reference chain bit-for-bit. Mounting is all-or-nothing per transformer: if any marked site fails the static guards (non-bf16 norm weight, hidden size above the kernel limit, ...) diff --git a/python/sglang/kernels/ops/diffusion/sites/helios_gated_residual_site.py b/python/sglang/kernels/ops/diffusion/sites/helios_gated_residual_site.py index 53d03f111f..83a7cb8852 100644 --- a/python/sglang/kernels/ops/diffusion/sites/helios_gated_residual_site.py +++ b/python/sglang/kernels/ops/diffusion/sites/helios_gated_residual_site.py @@ -7,8 +7,8 @@ The shared ``residual_gate_add`` kernel computes ``residual + update * gate`` in a single pass but requires one dtype, so the gate and update are first cast to BF16. That reordering of the FP32 multiply is numerically equivalent only at half-precision rounding level (not bit-exact), so the fusion is opt-in: -``quality="extra-high"`` and ``quality="high"`` mount it, while the default -``quality="lossless"`` keeps the reference FP32-multiply form bit-for-bit. +``quality="lossless"`` and ``quality="high"`` mount it, while the default +``quality="exact"`` keeps the reference FP32-multiply form bit-for-bit. """ from __future__ import annotations diff --git a/python/sglang/kernels/ops/diffusion/sites/lingbot_video_gated_residual_site.py b/python/sglang/kernels/ops/diffusion/sites/lingbot_video_gated_residual_site.py index c16b07c05f..56aadc90b6 100644 --- a/python/sglang/kernels/ops/diffusion/sites/lingbot_video_gated_residual_site.py +++ b/python/sglang/kernels/ops/diffusion/sites/lingbot_video_gated_residual_site.py @@ -7,8 +7,8 @@ the attention and FFN updates, where ``update`` (post-norm) and the per-token pass but requires one dtype, so the gate and update are first cast to BF16. That reordering of the FP32 multiply is numerically equivalent only at half-precision rounding level (not bit-exact), so the fusion is opt-in: -``quality="extra-high"`` and ``quality="high"`` mount it, while the default -``quality="lossless"`` keeps the reference FP32-multiply form bit-for-bit. +``quality="lossless"`` and ``quality="high"`` mount it, while the default +``quality="exact"`` keeps the reference FP32-multiply form bit-for-bit. """ from __future__ import annotations diff --git a/python/sglang/kernels/ops/diffusion/sites/lingbot_video_rmsnorm_site.py b/python/sglang/kernels/ops/diffusion/sites/lingbot_video_rmsnorm_site.py index 578d4f3d19..6c4a6f4863 100644 --- a/python/sglang/kernels/ops/diffusion/sites/lingbot_video_rmsnorm_site.py +++ b/python/sglang/kernels/ops/diffusion/sites/lingbot_video_rmsnorm_site.py @@ -7,7 +7,7 @@ the reference formula still promotes their multiplication through the FP32 hidden states. For ``quality="high"``, existing diffusion Triton RMSNorm kernels replace that chain. Wide rows with FP32 weights use the one-row ``norm_infer`` kernel; the remaining sites use the tiled one-pass kernel. Their -reduction order is not bit-exact, so the default ``quality="lossless"`` path +reduction order is not bit-exact, so the default ``quality="exact"`` path remains unchanged. """ diff --git a/python/sglang/kernels/ops/diffusion/sites/nvfp4_bias_gelu_site.py b/python/sglang/kernels/ops/diffusion/sites/nvfp4_bias_gelu_site.py index 0d800a1172..38866dab06 100644 --- a/python/sglang/kernels/ops/diffusion/sites/nvfp4_bias_gelu_site.py +++ b/python/sglang/kernels/ops/diffusion/sites/nvfp4_bias_gelu_site.py @@ -2,7 +2,7 @@ The fused JIT kernel is bit-exact with the local eager ``add + GELU`` chain for eligible ModelOpt FP4 linears. It still changes the model's kernel schedule, so -keep the default ``quality="lossless"`` path unchanged and mount this fast path +keep the default ``quality="exact"`` path unchanged and mount this fast path only for the existing ``quality="high"`` contract. """ diff --git a/python/sglang/kernels/ops/diffusion/sites/qwen_image_added_qkv_site.py b/python/sglang/kernels/ops/diffusion/sites/qwen_image_added_qkv_site.py index ab8234f330..5156c8df2c 100644 --- a/python/sglang/kernels/ops/diffusion/sites/qwen_image_added_qkv_site.py +++ b/python/sglang/kernels/ops/diffusion/sites/qwen_image_added_qkv_site.py @@ -2,7 +2,7 @@ Packing the three BF16 text projections into one GEMM changes the reduction association and is therefore not bit-exact. The packed weights stay resident -for checkpoint compatibility, but ``quality="lossless"`` applies their three +for checkpoint compatibility, but ``quality="exact"`` applies their three slices independently. ``quality="high"`` mounts the single-GEMM path. """ diff --git a/python/sglang/kernels/ops/diffusion/sites/sana_video_linear_attention_site.py b/python/sglang/kernels/ops/diffusion/sites/sana_video_linear_attention_site.py index fb4966bf08..d782f42cd4 100644 --- a/python/sglang/kernels/ops/diffusion/sites/sana_video_linear_attention_site.py +++ b/python/sglang/kernels/ops/diffusion/sites/sana_video_linear_attention_site.py @@ -6,7 +6,7 @@ requesting an FP32 output/accumulator from cuBLAS. The second GEMM stays in FP32. This removes two large dtype-conversion kernels and lets the first GEMM use BF16 Tensor Cores, at the cost of half-precision input rounding. -The default ``quality="lossless"`` path remains the original FP32-input chain +The default ``quality="exact"`` path remains the original FP32-input chain bit-for-bit. Only the single-batch CUDA layout used by native SANA-Video is eligible; unsupported dtypes and layouts fall back to the reference path. """ diff --git a/python/sglang/multimodal_gen/.agents/skills/sglang-diffusion-add-model/SKILL.md b/python/sglang/multimodal_gen/.agents/skills/sglang-diffusion-add-model/SKILL.md index d43593bdd1..18b4dd74ae 100644 --- a/python/sglang/multimodal_gen/.agents/skills/sglang-diffusion-add-model/SKILL.md +++ b/python/sglang/multimodal_gen/.agents/skills/sglang-diffusion-add-model/SKILL.md @@ -657,7 +657,7 @@ BCG validation must prove all of the following: For a non-bit-exact optimization, integrate through the request-scoped site framework under `sglang.kernels.ops.diffusion.sites`. Mark sites during model construction and let `QualityGatedFusion` mount them for both -`quality="extra-high"` and `quality="high"`; `quality="lossless"` must keep +`quality="lossless"` and `quality="high"`; `quality="lossless"` must keep the original code path. A high-only sparse, caching, or other approximate path must remain outside this fusion gate. Eligibility must be all-or-nothing for coupled sites and fail closed on dtype, @@ -666,7 +666,7 @@ guard/parity tests and a model wiring test instead of embedding request-policy branches throughout the DiT. Finally, use the benchmark/profile skill's `--quality-bcg-matrix` to run -same-GPU ABBA pairs for Eager/BCG at lossless/extra-high/high. Report denoise and saved +same-GPU ABBA pairs for Eager/BCG at exact/lossless/high. Report denoise and saved request e2e separately, require at least 1.5% repeated mean e2e improvement for an optimization PR, attach profile and generated-media A/B evidence, then delete the task-owned checkpoint cache and verify zero residual weight files diff --git a/python/sglang/multimodal_gen/.agents/skills/sglang-diffusion-benchmark-profile/SKILL.md b/python/sglang/multimodal_gen/.agents/skills/sglang-diffusion-benchmark-profile/SKILL.md index 3844bc779d..e908e7cd09 100644 --- a/python/sglang/multimodal_gen/.agents/skills/sglang-diffusion-benchmark-profile/SKILL.md +++ b/python/sglang/multimodal_gen/.agents/skills/sglang-diffusion-benchmark-profile/SKILL.md @@ -9,7 +9,7 @@ Use this skill when measuring denoise performance, finding the slow op, checking This skill is diagnosis-first. It owns: - checked-in denoise benchmark presets -- same-GPU quality/BCG applicability checks with repeated lossless, extra-high, and high rows +- same-GPU quality/BCG applicability checks with repeated exact, lossless, and high rows - perf dump collection and before/after comparison - `torch.profiler` trace capture and quick hotspot ranking - mapping hot kernels back to known fast paths and fusion families @@ -65,15 +65,15 @@ Always rule out these existing families first: - Z-Image bf16-native Triton RMSNorm scale/tanh-residual modulation - SANA packed self-attention Q/K/V and cross-attention K/V GEMMs - SANA-Video's packed projections and request-scoped BF16-input linear - attention at `quality=extra-high` or `quality=high`; keep the second attention GEMM in FP32 and - compare against `quality=lossless` before changing its precision further + attention at `quality=lossless` or `quality=high`; keep the second attention GEMM in FP32 and + compare against `quality=exact` before changing its precision further - SANA-Video reuse of SANA's bit-exact bias/activation, residual-gate, and LayerNorm-modulation fast paths before adding video-only kernels - MiniMax-H3 indexed modulation, fused QK norm + RoPE, packed Ulysses QKV, USP relayout, and batched TP AdaLN collectives - bit-exact diffusion adaLN modulation and fused LayerNorm + modulation for FLUX.1, GLM-Image, and SANA -- request-scoped DiT and VAE fast paths at `quality=extra-high` or `quality=high` +- request-scoped DiT and VAE fast paths at `quality=lossless` or `quality=high` - LingBot Video's default-on fused group-limited top-k expert selection before treating its router's `topk`/mask/gather chain as a new hotspot - Wan causal-VAE cache/padding and DupUp3D data-movement fusions @@ -96,12 +96,12 @@ controlled comparator, never for the eager ground truth. The legacy `--no-torch-compile` spelling remains accepted but is redundant. For kernel/BCG discovery, run `--quality-bcg-matrix`. It executes Eager/BCG as -A-B-B-A at `lossless`, then repeats the pair at `extra-high` and `high`, on -one locked GPU set and one isolated checkpoint cache. The extra-high/high+BCG +A-B-B-A at `exact`, then repeats the pair at `lossless` and `high`, on +one locked GPU set and one isolated checkpoint cache. The lossless/high+BCG rows are applicability checks, not presumed-valid performance cells. A BCG row is invalid unless the log contains `[Diffusion BCG] captured` and contains no support-disable, capture-failure, serving-signature-miss, or late quality-fusion marker. In -particular, a request-scoped DiT fusion mounted after lossless warmup capture +particular, a request-scoped DiT fusion mounted after exact warmup capture would be bypassed by replay; reject that row even when capture and signature checks pass. For video presets, the helper declares both the request resolution and `--warmup-num-frames` so the synthetic BCG warmup captures the requested @@ -122,7 +122,7 @@ its normal cleanup finally block without modifying the seed cache. Keep prompt, negative prompt, seed, shape, steps, guidance, dtype, topology, and residency fixed. Lossless comparisons require byte-identical artifacts. -For `quality=extra-high` and `quality=high`, report aggregate and worst-frame SSIM/PSNR; the repository +For `quality=lossless` and `quality=high`, report aggregate and worst-frame SSIM/PSNR; the repository defaults are 0.95/28 dB for images and 0.92/24 dB for video unless the model's checked-in consistency metadata defines a different threshold. A performance PR needs repeated saved-request e2e improvement of at least 1.5%, a diff --git a/python/sglang/multimodal_gen/.agents/skills/sglang-diffusion-benchmark-profile/benchmark-and-profile.md b/python/sglang/multimodal_gen/.agents/skills/sglang-diffusion-benchmark-profile/benchmark-and-profile.md index efef0fe806..7517c94f46 100644 --- a/python/sglang/multimodal_gen/.agents/skills/sglang-diffusion-benchmark-profile/benchmark-and-profile.md +++ b/python/sglang/multimodal_gen/.agents/skills/sglang-diffusion-benchmark-profile/benchmark-and-profile.md @@ -139,9 +139,9 @@ The helper defaults to eager. Add `--torch-compile` only for a labeled compile control. `--no-torch-compile` remains accepted for compatibility but is no longer required. -Run one explicit quality or BCG comparator with `--quality {lossless,extra-high,high}` and +Run one explicit quality or BCG comparator with `--quality {exact,lossless,high}` and `--breakable-cuda-graph`. BCG and `torch.compile` are intentionally mutually -exclusive in this helper. An extra-high/high+BCG command is only a compatibility probe: +exclusive in this helper. An lossless/high+BCG command is only a compatibility probe: it is invalid if request-scoped DiT fusions mount after the lossless warmup graphs were captured. When a preset has explicit width and height, the helper declares that same `--warmup-resolutions` value automatically. Video presets @@ -150,16 +150,16 @@ with an explicit frame count also declare the matching `--warmup-num-frames`: ```bash PYTHONPATH=python python3 "$BENCH_PY" \ --model longcat-image \ - --quality extra-high \ + --quality lossless \ --breakable-cuda-graph \ - --label bcg-extra-high \ + --label bcg-lossless \ --output-dir "${BENCH_DIR}" ``` For optimization discovery, use the full repeated matrix. It runs -Eager/BCG/BCG/Eager at `lossless`, then the same sequence at `extra-high` and +Eager/BCG/BCG/Eager at `exact`, then the same sequence at `lossless` and `high`, while holding one GPU set and one isolated checkpoint cache. The -extra-high/high+BCG cells test +lossless/high+BCG cells test whether the combination is actually supported; do not average them when the runtime rejects the combination or the helper detects a late quality-fusion mount. The helper hashes every generated image, video, audio, or 3D mesh @@ -321,13 +321,13 @@ Use the preset categories this way: | `qwen-edit-base` | `Qwen/Qwen-Image-Edit` | No | Covers the original native `QwenImageEditPipelineConfig`, which is distinct from the 2509/2511 edit-plus paths; public SGLang edit fixture, 1024x1024. | | `qwen-image-layered` | `Qwen/Qwen-Image-Layered` | No | Native layered-image path using the same public reference image and four-frame request as the GPU server case, at the registered 640x640 canvas. | | `stable-diffusion-3.5-medium` | `stabilityai/stable-diffusion-3.5-medium-diffusers` | No | Representative native `StableDiffusion3PipelineConfig` path at 1024x1024. The repository is gated, so export `HF_TOKEN`; an unauthenticated run is a recorded access blocker, not model evidence. | -| `sana-video` | `Efficient-Large-Model/SANA-Video_2B_480p_diffusers` | No | CI-sized T2V baseline: 832x480, 17 frames, 8 steps, guidance 6.0. The BCG comparator declares the same 17-frame warmup shape. Compare all three tiers; `extra-high` and `high` enable the BF16-input first linear-attention GEMM while retaining FP32 output and the FP32 second GEMM. | +| `sana-video` | `Efficient-Large-Model/SANA-Video_2B_480p_diffusers` | No | CI-sized T2V baseline: 832x480, 17 frames, 8 steps, guidance 6.0. The BCG comparator declares the same 17-frame warmup shape. Compare all three tiers; `lossless` and `high` enable the BF16-input first linear-attention GEMM while retaining FP32 output and the FP32 second GEMM. | | `sana-wm-bidirectional` | `Efficient-Large-Model/SANA-WM_bidirectional` | No | Dense two-stage TI2V baseline at the native 1280x704 shape, 49 frames, 16 fps, 20 steps, guidance 4.5, and a 48-frame forward/left action program. Uses the shared cat fixture. | | `sana-wm-streaming` | `Efficient-Large-Model/SANA-WM_streaming` | No | Matching offline chunk-causal two-stage baseline with the streaming DiT and chunked refiner enabled; uses the same shape, fixture, seed, and camera action for comparison. | | `lingbot-video-moe` | `robbyant/lingbot-video-moe-30b-a3b` | No | One-GPU eager baseline using the CI structured-JSON caption, 384x640, 17 frames, 12 steps, and text-encoder CPU offload. | | `lingbot-world` | `robbyant/lingbot-world-fast-diffusers` | No | One-H200 offline single-chunk profile for the registered causal DMD path: 832x480x9, four steps, guidance 1.0, the shared image fixture, and forward-camera actions for all nine frames. Keep stateful websocket latency as a separate metric. | | `lingbot-world-v2` | `robbyant/lingbot-world-v2-14b-causal-fast-diffusers` | No | Matching controlled single-chunk profile for the separately registered v2 checkpoint. The fixed shape, action program, and schedule make v1/v2 hotspot comparisons reproducible without presenting one-chunk e2e as stateful realtime latency. | -| `fastwan21-t2v-1.3b` | `FastVideo/FastWan2.1-T2V-1.3B-Diffusers` | No | One-GPU 832x480, 61-frame, 3-step DMD baseline. The preset pins manual mode with a resident DiT so lossless/extra-high/high comparisons do not measure an offload-policy change. | +| `fastwan21-t2v-1.3b` | `FastVideo/FastWan2.1-T2V-1.3B-Diffusers` | No | One-GPU 832x480, 61-frame, 3-step DMD baseline. The preset pins manual mode with a resident DiT so exact/lossless/high comparisons do not measure an offload-policy change. | | `wan21-t2v-1.3b` | `Wan-AI/Wan2.1-T2V-1.3B-Diffusers` | No | Registered one-GPU 832x480, 81-frame Wan2.1 baseline at 50 steps and guidance 3.0. Keep it separate from FastWan and TurboWan because the longer schedule changes the end-to-end weight of VAE optimizations. | | `wan21-t2v-14b` | `Wan-AI/Wan2.1-T2V-14B-Diffusers` | No | Cookbook-aligned four-GPU CFG/Ulysses baseline at 832x480, 81 frames, 50 steps, and guidance 5.0. Text encoding stays CPU-offloaded as in the documented deployment command. | | `wan21-i2v-14b-480p` | `Wan-AI/Wan2.1-I2V-14B-480P-Diffusers` | No | Four-GPU CFG/Ulysses image-conditioned baseline at 832x480, 81 frames, 50 steps, and guidance 5.0. Uses the shared cat fixture and its motion prompt. | @@ -669,8 +669,8 @@ a zero-exit run if either the requested perf dump or generated media is absent: some generation failures are reported through the response payload without a nonzero process exit. -For `quality=lossless`, compare saved artifact hashes and require byte equality -for a claimed lossless fast path or BCG change. For `quality=extra-high` and +For `quality=exact`, compare saved artifact hashes and require byte equality +for a claimed lossless fast path or BCG change. For `quality=lossless` and `quality=high`, keep the lossless artifact as ground truth and report both aggregate and worst-frame SSIM/PSNR. Repository defaults are SSIM 0.95 / PSNR 28 dB for images and SSIM @@ -791,7 +791,7 @@ the known mainline families. | `to_q -> to_k -> to_v` on NVFP4 or Nunchaku FLUX-family checkpoints | Treat as a packed-QKV fast-path miss or checkpoint-format mismatch | | `rmsnorm_scale` or `rmsnorm_tanh_residual` missing on Z-Image | Check the bf16-native Triton eligibility guards before proposing a new fusion | | FLUX.1, GLM-Image, or SANA shows separate LayerNorm plus adaLN elementwise kernels | Check the bit-exact `modulate_scale_shift` and `fused_layernorm_modulate` guards/self-test before proposing another norm fusion | -| `quality=extra-high` or `quality=high` shows the same FLUX/GLM DiT or FLUX-family/Wan VAE chain as `lossless` | Check whether the request-scoped quality gate mounted and whether every site passed its all-or-nothing compatibility checks | +| `quality=lossless` or `quality=high` shows the same FLUX/GLM DiT or FLUX-family/Wan VAE chain as `exact` | Check whether the request-scoped quality gate mounted and whether every site passed its all-or-nothing compatibility checks | | LTX-2 split RoPE appears as a long PyTorch elementwise chain | Check the `apply_ltx2_split_rotary_emb` Triton path and its shape guards | | Wan decode is dominated by causal `cat + pad + contiguous`, feature-cache copies, or `repeat_interleave + permute + add` | Check the bit-exact Wan causal-cache and DupUp3D data-movement kernels before writing a new decoder kernel | | masked attention spends time packing/unpacking Q/K/V | Check whether fused varlen USP pack/scatter should have engaged | @@ -829,7 +829,7 @@ This skill intentionally stops here. It tells you whether you are looking at: - [ ] `compare_perf.py` table generated - [ ] one representative `torch.profiler` trace saved - [ ] hotspot classified against `existing-fast-paths.md` -- [ ] lossless artifact hash is exact; extra-high/high aggregate and worst-frame SSIM/PSNR pass the checked-in threshold +- [ ] lossless artifact hash is exact; lossless/high aggregate and worst-frame SSIM/PSNR pass the checked-in threshold - [ ] reference image or start/middle/end video contact sheet checked visually - [ ] any PR claim has repeated saved-request e2e improvement >= 1.5% - [ ] task-owned checkpoint cache cleaned and ledger shows zero residual weight files diff --git a/python/sglang/multimodal_gen/.agents/skills/sglang-diffusion-benchmark-profile/existing-fast-paths.md b/python/sglang/multimodal_gen/.agents/skills/sglang-diffusion-benchmark-profile/existing-fast-paths.md index 680f3c5d3c..bac7107d0f 100644 --- a/python/sglang/multimodal_gen/.agents/skills/sglang-diffusion-benchmark-profile/existing-fast-paths.md +++ b/python/sglang/multimodal_gen/.agents/skills/sglang-diffusion-benchmark-profile/existing-fast-paths.md @@ -110,12 +110,12 @@ framework-specific optimization workflow. check dtype, alignment, shape, BCG/compile context, and the one-time equality self-test before proposing another fusion. -4. Request-scoped fusion gates at `quality=extra-high` or `quality=high` +4. Request-scoped fusion gates at `quality=lossless` or `quality=high` - Locations: `quality_gate.py`, `fused_ln_modulate.py`, `denoising.py`, `decoding.py`, `fast_path_gate.py`, `flux2_vae_cuda_opt.py`, and `wan_vae_cuda_opt.py`. - Behavior: `quality="lossless"` is the default exact reference path. - `quality="extra-high"` and `quality="high"` mount the same validated but + `quality="lossless"` and `quality="high"` mount the same validated but non-bit-exact DiT fusions and decode-scoped VAE rewrites. `high` is cumulative and may additionally enable model-owned approximate paths. Mounting is all-or-nothing per @@ -397,13 +397,13 @@ framework-specific optimization workflow. **Request-Scoped DiT Fusions with Breakable CUDA Graphs** -- DiT sites at `quality=extra-high` or `quality=high` are mounted at a request boundary. BCG warmup uses +- DiT sites at `quality=lossless` or `quality=high` are mounted at a request boundary. BCG warmup uses the model's lossless sampling default unless a quality-aware graph variant was captured explicitly. - A graph captured before the request-quality mount retains the lossless module branches. Replaying it after the mount silently bypasses the requested fused kernels even when the tensor signature matches. -- Workflow rule: an extra-high/high+BCG cell is valid only when the model has no +- Workflow rule: an lossless/high+BCG cell is valid only when the model has no request-scoped DiT quality sites, or when logs prove those sites were mounted before the matching graph capture. A mount after `[Diffusion BCG] captured` invalidates the row; do not use its latency or output as request-quality @@ -434,7 +434,7 @@ framework-specific optimization workflow. selector when its CUDA layout guard passes. Check that guard before treating the `topk`/mask chain as a new opportunity. Its fused Triton RMSNorm row kernels remain request-gated by weight dtype and hidden size at - `quality=extra-high` and `quality=high`; check those separate quality-site + `quality=lossless` and `quality=high`; check those separate quality-site guards before treating an expanded `pow/mean/rsqrt` chain as new work. - LTX-2.5 reuses the mature LTX-2 DiT paths. Treat the optional diffusion decoder separately: confirm NATTEN `na3d` is active, then inspect its @@ -448,7 +448,7 @@ framework-specific optimization workflow. - AdaLN modulation: `LayerNormScaleShift`, `RMSNormScaleShift`, `ScaleResidual*` in `layernorm.py`. - Bit-exact adaLN modulation / LayerNorm folding: `modulate_scale_shift` and `fused_layernorm_modulate` through `flux.py`, `glm_image.py`, and `sana.py`. -- Request-scoped extra-high/high acceleration: `QualityGatedFusion` in +- Request-scoped lossless/high acceleration: `QualityGatedFusion` in `quality_gate.py`, `_maybe_toggle_quality_fusions` in `denoising.py`, and `use_vae_fast_path` in `decoding.py`. - Bit-exact first-sight verify/disable: `BitExactFusionGate` in @@ -466,7 +466,7 @@ framework-specific optimization workflow. - LingBot Video MoE routing: `LingBotVideoRouter._group_limited_topk` uses `group_limited_topk` through the diffusion facade when the CUDA layout guard passes, otherwise it retains the eager reference chain. -- LTX2 RMSNorm+modulate and FFN GELU epilogue under `quality="extra-high"` and `quality="high"`: +- LTX2 RMSNorm+modulate and FFN GELU epilogue under `quality="lossless"` and `quality="high"`: `mark_ltx2_rms_norm_modulate_site` / `fused_ltx2_rms_norm_modulate` in `kernels/ops/diffusion/sites/ltx2_rmsnorm_modulate_site.py` (mount-based `QualityGatedFusion`, not a first-sight `BitExactFusionGate` — the fused diff --git a/python/sglang/multimodal_gen/.agents/skills/sglang-diffusion-benchmark-profile/scripts/bench_diffusion_denoise.py b/python/sglang/multimodal_gen/.agents/skills/sglang-diffusion-benchmark-profile/scripts/bench_diffusion_denoise.py index 322001b246..d0c34cd88d 100755 --- a/python/sglang/multimodal_gen/.agents/skills/sglang-diffusion-benchmark-profile/scripts/bench_diffusion_denoise.py +++ b/python/sglang/multimodal_gen/.agents/skills/sglang-diffusion-benchmark-profile/scripts/bench_diffusion_denoise.py @@ -16,7 +16,7 @@ Usage: # Opt in to a compile control (presets are eager by default) python3 python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-benchmark-profile/scripts/bench_diffusion_denoise.py --model flux --torch-compile - # Check Eager/BCG at every request quality on one GPU set; extra-high/high + # Check Eager/BCG at every request quality on one GPU set; lossless/high # + BCG are invalid when request-scoped DiT fusions mount only after the # lossless graph capture. python3 python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-benchmark-profile/scripts/bench_diffusion_denoise.py --model sana-video --quality-bcg-matrix --model-cache-root /task/model-caches --cleanup-model-cache @@ -81,7 +81,7 @@ DIFFUSERS_FALLBACK_SIGNALS = ( "using diffusers backend", "loaded diffusers pipeline", ) -BENCHMARK_QUALITY_LEVELS = ("lossless", "extra-high", "high") +BENCHMARK_QUALITY_LEVELS = ("exact", "lossless", "high") BCG_CAPTURE_SIGNAL = "[diffusion bcg] captured" BCG_INVALID_SIGNALS = ( "[diffusion bcg] capture failed", @@ -92,14 +92,14 @@ BCG_INVALID_SIGNALS = ( ) BCG_LATE_QUALITY_FUSION_SIGNAL = "quality fusion mounted after BCG capture" QUALITY_BCG_ABBA_MATRIX = ( + ("eager-exact-a", "exact", False), + ("bcg-exact-a", "exact", True), + ("bcg-exact-b", "exact", True), + ("eager-exact-b", "exact", False), ("eager-lossless-a", "lossless", False), ("bcg-lossless-a", "lossless", True), ("bcg-lossless-b", "lossless", True), ("eager-lossless-b", "lossless", False), - ("eager-extra-high-a", "extra-high", False), - ("bcg-extra-high-a", "extra-high", True), - ("bcg-extra-high-b", "extra-high", True), - ("eager-extra-high-b", "extra-high", False), ("eager-high-a", "high", False), ("bcg-high-a", "high", True), ("bcg-high-b", "high", True), @@ -1897,7 +1897,7 @@ def _run_benchmark_once_impl( if BCG_CAPTURE_SIGNAL in lower_line: bcg_capture_detected = True if ( - quality in {"extra-high", "high"} + quality in {"lossless", "high"} and breakable_cuda_graph and bcg_capture_detected and "mounted " in lower_line @@ -2315,7 +2315,7 @@ def main(): "--quality-bcg-matrix", action="store_true", help=( - "Run lossless/extra-high/high Eager-vs-BCG as three ABBA pairs " + "Run exact/lossless/high Eager-vs-BCG as three ABBA pairs " "on one GPU set " "and one task-owned model cache." ), diff --git a/python/sglang/multimodal_gen/.agents/skills/sglang-diffusion-performance/SKILL.md b/python/sglang/multimodal_gen/.agents/skills/sglang-diffusion-performance/SKILL.md index c248c241a9..ac4c21beea 100644 --- a/python/sglang/multimodal_gen/.agents/skills/sglang-diffusion-performance/SKILL.md +++ b/python/sglang/multimodal_gen/.agents/skills/sglang-diffusion-performance/SKILL.md @@ -163,7 +163,7 @@ These options **trade output quality** for speed or VRAM savings. Results will d | Option | CLI Flag / Env Var | What It Does | Speedup | Quality Impact / Limitations | |---|---|---|---|---| -| **Request Quality Fast Paths** | `--quality {extra-high,high}` (`lossless` is default) | `extra-high` mounts only request-gated DiT/VAE fusions. `high` includes that complete set and may add model-owned approximate paths such as Cache-DiT or lower-precision decode. | Model- and shape-specific | Support is per model and may be a no-op. Keep `--quality lossless` as the A/B ground truth, then compare `extra-high` before `high` to isolate fusion wins. Report aggregate and worst-frame SSIM/PSNR for every non-bit-exact path; defaults are 0.95/28 dB for images and 0.92/24 dB for video unless checked-in model metadata overrides them. Do not confuse this with `--output-quality`, which controls file compression. | +| **Request Quality Fast Paths** | `--quality {lossless,high}` (`exact` is default) | `lossless` mounts only request-gated DiT/VAE fusions. `high` includes that complete set and may add model-owned approximate paths such as Cache-DiT or lower-precision decode. | Model- and shape-specific | Support is per model and may be a no-op. Keep `--quality lossless` as the A/B ground truth, then compare `lossless` before `high` to isolate fusion wins. Report aggregate and worst-frame SSIM/PSNR for every non-bit-exact path; defaults are 0.95/28 dB for images and 0.92/24 dB for video unless checked-in model metadata overrides them. Do not confuse this with `--output-quality`, which controls file compression. | | **Approximate Attention** | Server-wide: `--attention-backend sage_attn` / `sage_attn_3` / `sliding_tile_attn` / `video_sparse_attn` / `sparse_video_gen_2_attn` / `vmoba_attn` / `sla_attn` / `sage_sla_attn`. Per-request (dense drop-ins only): `--attention-backend-override sage_attn` sampling param / API `extra_body` — valid values `fa`, `torch_sdpa`, `sage_attn`, `sage_attn_3`; rejected (with a log) under BCG, torch.compile, sparse server backends, or a non-ring-capable target with ring parallelism. | Replaces exact attention with approximate or sparse variants. `sage_attn`: INT8/FP8 quantized Q·K; `sliding_tile_attn`: spatial-temporal tile skipping; others: model-specific sparse patterns. | ~1.5–2x on attention (varies by backend) | Quality degradation varies by backend and model. `sage_attn` is the most general; sparse backends (`sliding_tile_attn`, `video_sparse_attn`, etc.) are video-model-specific, may require config files (e.g. `--mask-strategy-file-path` for STA), and are server-level only. Requires corresponding packages installed. | | **Cache-DiT** | Native: per-request `--enable-cache-dit true\|false` + `--cache-dit-params ` (sampling params; also via API `extra_body`). `SGLANG_CACHE_DIT_ENABLED` / `SGLANG_CACHE_DIT_*` env vars are the server-wide defaults for requests that leave them unset. Diffusers backend: `--backend diffusers --cache-dit-config ` | Caches intermediate residuals across denoising steps and skips redundant computations via DBCache, TaylorSeer, and optional SCM. | ~1.5-2x on supported models | Quality depends on cache policy. Compatible with `--dit-layerwise-offload`: skipped blocks are not streamed, and the first layer after a skip may sync-load. Models that touch every layer before the block loop (for example a full-stack AdaLN prepass) must keep that prepass off while caching. Do not pass `--cache-dit-config` for native SGLang tuning unless you are intentionally using the diffusers backend flow. | | **CFG Gating** | Per-request `--cfg-gate-step 0.5` (sampling param; also via API `extra_body`). `SGLANG_DIFFUSION_CFG_GATE_STEP` is the server-wide default (1.0 = off). | After the given fraction of denoising steps, reuses the cached cond-uncond residual instead of running the unconditional branch each step. | Up to ~2x on the gated tail of CFG models (skips one of two branches) | Lossy; no-op without classifier-free guidance or with `--enable-cfg-parallel`. Lower fractions gate earlier and drift more. | @@ -373,7 +373,7 @@ For video, also match the captured frame and conditioning shape; `WxH` alone does not prove replay. For a repeated discovery sweep, use the benchmark/profile helper. This runs -lossless, extra-high, and high Eager/BCG ABBA pairs on one GPU set, then deletes the +lossless, lossless, and high Eager/BCG ABBA pairs on one GPU set, then deletes the model group cache once: ```bash @@ -391,8 +391,8 @@ sglang generate --model-path \ --perf-dump-path baseline.json --save-output sglang generate --model-path \ - --quality extra-high --prompt "..." --seed 42 \ - --perf-dump-path quality-extra-high.json --save-output + --quality lossless --prompt "..." --seed 42 \ + --perf-dump-path quality-lossless.json --save-output sglang generate --model-path \ --quality high --prompt "..." --seed 42 \ @@ -400,7 +400,7 @@ sglang generate --model-path \ ``` Keep every other flag fixed and compare the generated artifact as well as the -perf dumps. `high` must retain every fusion observed under `extra-high`. If the +perf dumps. `high` must retain every fusion observed under `lossless`. If the model has no registered request-gated or high-only sites, either tier may be a no-op. @@ -510,7 +510,7 @@ Use these as first commands to benchmark, not as universal winners. | Z-Image / Z-Image-Turbo | 1024x1024, runtime-default steps/guidance, 1 GPU | `--enable-torch-compile --warmup-mode request` | Keep base Z-Image separate from Turbo: base uses 50-step CFG defaults, Turbo uses 9-step zero-CFG defaults. Mainline has bf16-native Triton RMSNorm scale and tanh-residual fusions. | | Wan2.2 A14B T2V/I2V | 1280x720, 81 frames | Nightly: `--num-gpus 4 --enable-cfg-parallel --ulysses-degree 2 --text-encoder-cpu-offload --pin-cpu-memory` | For lowest latency, also benchmark pure Ulysses on the same GPUs. | | Wan2.2 TI2V 5B | 1280x720, 81 frames, 1 GPU | `--enable-torch-compile --warmup-mode request` | Keep the input image and motion prompt fixed when comparing sparse attention or Cache-DiT. | -| Wan2.1 / FastWan / TurboWan variants | 480p or 720p video, family defaults | Compare `--quality lossless`, `--quality extra-high`, and `--quality high`, then try `--enable-torch-compile --warmup-mode request`; add `--ulysses-degree` / CFG parallel only after measuring | `extra-high` and `high` mount the Wan FFN cublasLt/NVFP4 GELU epilogues and the Wan VAE RMSNorm+SiLU fast path when their guards pass; validate video quality against lossless. Current registry includes Wan2.1, FastWan2.1, FastWan2.2 TI2V, TurboWan2.1, TurboWan2.2 I2V, and Wan2.1-Fun InP. Use the compatibility matrix and benchmark presets before choosing topology. | +| Wan2.1 / FastWan / TurboWan variants | 480p or 720p video, family defaults | Compare `--quality lossless`, `--quality lossless`, and `--quality high`, then try `--enable-torch-compile --warmup-mode request`; add `--ulysses-degree` / CFG parallel only after measuring | `lossless` and `high` mount the Wan FFN cublasLt/NVFP4 GELU epilogues and the Wan VAE RMSNorm+SiLU fast path when their guards pass; validate video quality against lossless. Current registry includes Wan2.1, FastWan2.1, FastWan2.2 TI2V, TurboWan2.1, TurboWan2.2 I2V, and Wan2.1-Fun InP. Use the compatibility matrix and benchmark presets before choosing topology. | | Cosmos3 Nano / Super | T2I: 1024x1024 with `--num-frames 1`; T2V/I2V: 480p/720p video | Start with `--performance-mode auto --warmup-mode request`; use `SGLANG_DISABLE_COSMOS3_GUARDRAILS=1` only for benchmark isolation, and compare compile separately | One checkpoint serves T2I/T2V/I2V. Mode is request-driven: `num_frames == 1` means T2I, `--image-path` means I2V. On GPUs with at least 120 GiB available, auto mode keeps the Cosmos3 DiT and VAE resident for every checkpoint in the family; a 1xH200 832x480x9f, 4-step eager ABBA reduced e2e from 1.576 to 0.428 seconds with exact output parity. Cosmos3 runs one DiT per pipeline, so component offload above that threshold only buys a DiT copy out to host memory and back per request -- it cost Cosmos3-Super 720p 81f T2V ~4s of ~115s on 2xH200. | | Cosmos3 Edge / distilled Super | Edge T2I: 640x640, 35 steps, 1 GPU; distilled Super T2I: 640x640, fixed 4-step schedule, 4 GPUs | Start eager with `--performance-mode manual`; use `SGLANG_DISABLE_COSMOS3_GUARDRAILS=1` only for benchmark isolation | Edge is trained for 256p/480p shapes. Distilled checkpoints own their sigma schedule and force guidance 1.0; do not override steps or flow shift. Do not retry the closed experimental Cosmos BCG path without a new lifecycle design. | | Ideogram 4 FP8/NVFP4 | 1024x1024, native preset defaults | `--enable-torch-compile --warmup-mode request` | Do not set `--num-inference-steps` or `--guidance-scale` directly unless you also update the Ideogram preset; sampling params derive them from `preset`. | @@ -523,7 +523,7 @@ Use these as first commands to benchmark, not as universal winners. | JoyAI-Image-Edit | 1024-class TI2I, 40 steps, guidance 4.0 | `--backend=sglang --num-gpus 2 --enable-cfg-parallel --ulysses-degree 1 --enable-torch-compile --warmup-mode request --dit-layerwise-offload false --dit-cpu-offload false` | Newly supported image-edit path. Keep the input image, prompt, seed, and output size fixed; 2-GPU CFG parallel is the validated H100 starting point. | | FireRed-Image-Edit 1.0 / 1.1 | 1024x1024 image edit, 40 steps, guidance 4.0 | `--backend=sglang --num-gpus 2 --enable-cfg-parallel --ulysses-degree 1 --enable-torch-compile --warmup-mode request --dit-layerwise-offload false --dit-cpu-offload false` | Uses the native `QwenImageEditPlusPipeline` path. 2-GPU CFG parallel is the validated H100 starting point; benchmark 1.0 and 1.1 separately because checkpoint differences can change denoise latency. | | Hunyuan3D-2 shape | Shape generation, 50 steps, guidance 5.0 | `--backend=sglang --enable-torch-compile --warmup-mode request --dit-layerwise-offload false --dit-cpu-offload false` | Focus on `Hunyuan3DShapeDenoisingStage`; keep mesh export/paint timings separate from denoise. | -| LingBot Video MoE 30B | 384x640, 17 frames, 12 steps for the current GPU case | `--model-path robbyant/lingbot-video-moe-30b-a3b --text-encoder-cpu-offload` | Native T2V path. Prompts are structured JSON captions, not raw free text; keep that contract when comparing latency or quality. Current main can mount the fused Triton RMSNorm path at `quality=extra-high` or `quality=high`; keep `lossless` as the reference. `--text-encoder-cpu-offload` targets memory-bound multi-GPU or small-VRAM cards; on a single large-VRAM GPU (e.g. 275 GB B300) the whole model stays resident (~73 GB peak), so dropping the flag removes H2D/D2H traffic and was 8% faster end to end (3.80s -> 3.49s, bit-identical). | +| LingBot Video MoE 30B | 384x640, 17 frames, 12 steps for the current GPU case | `--model-path robbyant/lingbot-video-moe-30b-a3b --text-encoder-cpu-offload` | Native T2V path. Prompts are structured JSON captions, not raw free text; keep that contract when comparing latency or quality. Current main can mount the fused Triton RMSNorm path at `quality=lossless` or `quality=high`; keep `exact` as the reference. `--text-encoder-cpu-offload` targets memory-bound multi-GPU or small-VRAM cards; on a single large-VRAM GPU (e.g. 275 GB B300) the whole model stays resident (~73 GB peak), so dropping the flag removes H2D/D2H traffic and was 8% faster end to end (3.80s -> 3.49s, bit-identical). | | MOVA / Helios / LingBot World | Start from the family's cookbook page — it pins the GPU count and parallelism these realtime paths need (e.g. LingBot World is served with `--num-gpus 4 --ulysses-degree 4`, LingBot World 2.0 with 8) | `--enable-torch-compile --warmup-mode request`; pin offload and topology flags explicitly | These video/realtime families have model-specific stages and condition handling. Realtime keeps up with playback only at the documented GPU count, so do not tune down from it by accident. For LingBot World causal serving, keep `--kv-cache-quant off` as the exact cache baseline before testing INT4/INT2. | ## Historical PR Watchlist diff --git a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/core/server_api.py b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/core/server_api.py index 602eca0c99..944d93ca82 100644 --- a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/core/server_api.py +++ b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/core/server_api.py @@ -99,8 +99,8 @@ class SGLDiffusionServerAPI: seed: Random seed for reproducible generation enable_teacache: Enable TEA cache acceleration response_format: Response format ("b64_json" or "url") - quality: Request optimization tier ("auto", "lossless", - "extra-high", "high") - only for generation + quality: Request optimization tier ("auto", "exact", + "lossless", "high") - only for generation style: Image style ("vivid" or "natural") - only for generation background: Background type ("auto", "transparent", "opaque") output_format: Output format ("png", "jpeg", "webp") diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/minimax_h3.py b/python/sglang/multimodal_gen/configs/pipeline_configs/minimax_h3.py index 7e30540a90..5d20b4730d 100644 --- a/python/sglang/multimodal_gen/configs/pipeline_configs/minimax_h3.py +++ b/python/sglang/multimodal_gen/configs/pipeline_configs/minimax_h3.py @@ -342,7 +342,7 @@ class FastH3PipelineConfig(MiniMaxH3PipelineConfig): raise ValueError( 'quality="high" is audited only for the base MiniMax-H3 50-step ' "4xH200 deployment; the FastH3 4-step distilled checkpoint has no " - 'audited high-quality deployment. Use quality="lossless".' + 'audited high-quality deployment. Use quality="exact".' ) def validate_server_args(self, server_args) -> None: diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/minimax_h3_vdn.py b/python/sglang/multimodal_gen/configs/pipeline_configs/minimax_h3_vdn.py index 9a9e0f0c68..a8285936e0 100644 --- a/python/sglang/multimodal_gen/configs/pipeline_configs/minimax_h3_vdn.py +++ b/python/sglang/multimodal_gen/configs/pipeline_configs/minimax_h3_vdn.py @@ -23,7 +23,7 @@ class VDNH3PipelineConfig(MiniMaxH3PipelineConfig): raise ValueError( 'quality="high" is audited only for the base MiniMax-H3 50-step ' "4xH200 deployment; the VDN-H3 8-step hybrid checkpoint has no " - 'audited high-quality deployment. Use quality="lossless".' + 'audited high-quality deployment. Use quality="exact".' ) def validate_server_args(self, server_args) -> None: diff --git a/python/sglang/multimodal_gen/configs/sample/sampling_params.py b/python/sglang/multimodal_gen/configs/sample/sampling_params.py index 652abefe5d..6db75b1d95 100644 --- a/python/sglang/multimodal_gen/configs/sample/sampling_params.py +++ b/python/sglang/multimodal_gen/configs/sample/sampling_params.py @@ -53,12 +53,15 @@ def generate_request_id() -> str: return str(uuid.uuid4()) -# Validated request-level quality levels, ordered from the strictest numerical -# contract to the broadest optimization set. "lossless" keeps the exact -# reference path; "extra-high" adds only request-gated kernel fusions; "high" -# is cumulative and may also enable model-owned approximate optimizations. -QUALITY_LEVELS: tuple[str, ...] = ("lossless", "extra-high", "high") -KERNEL_FUSION_QUALITY_LEVELS = frozenset({"extra-high", "high"}) +# Request-level quality levels, ordered from the strictest numerical contract to +# the broadest optimization set; each level includes the fast paths of the ones +# before it: +# - "exact": bit-identical to the reference path in the same environment +# - "lossless": the same math; fast paths may change rounding order or position +# - "high": may also change what is computed (precision, sparsity, reuse) +QUALITY_LEVELS: tuple[str, ...] = ("exact", "lossless", "high") +# "extra-high" is the pre-rename name of "lossless" +QUALITY_ALIASES: dict[str, str] = {"extra-high": "lossless"} # libx264 presets, fastest first: each spends more encode time for a smaller # file at the same CRF. X264_PRESETS: tuple[str, ...] = ( @@ -126,9 +129,20 @@ def resolve_skip_softmax_params( return SkipSoftmaxParams(float(threshold), start_step) -def quality_allows_kernel_fusions(quality: str) -> bool: - """Return whether a quality level includes request-gated kernel fusions.""" - return quality in KERNEL_FUSION_QUALITY_LEVELS +def normalize_quality(quality: str) -> str: + """Return the canonical quality level for ``quality``, resolving aliases.""" + level = QUALITY_ALIASES.get(quality, quality) + if level not in QUALITY_LEVELS: + raise ValueError( + f"quality must be one of {[*QUALITY_LEVELS, *QUALITY_ALIASES]}, " + f"got {quality!r}" + ) + return level + + +def quality_allows(quality: str, tier: str) -> bool: + """Whether a request at ``quality`` may run a fast path of class ``tier``.""" + return QUALITY_LEVELS.index(quality) >= QUALITY_LEVELS.index(tier) def _sanitize_filename(name: str, replacement: str = "_", max_length: int = 150) -> str: @@ -248,18 +262,20 @@ class SamplingParams: output_compression: int | None = None # MP4 encode speed; None keeps the server's default preset x264_preset: str | None = None - # Model-owned, request-scoped quality level. + # Model-owned, request-scoped quality level (see QUALITY_LEVELS). # - # - "lossless" (default): the exact reference path. Output is expected to - # be bit-identical to the HF reference implementation and to pass the - # CI golden/ground-truth comparisons. - # - "extra-high": add only validated kernel fusions. These may change - # half-precision rounding order, so output is not bit-exact versus the - # reference, but this tier does not itself enable sparse or approximate - # optimizations. - # - "high": include every "extra-high" fusion and allow model-owned - # approximate optimizations such as sparse computation or feature - # caching. These paths require model-specific quality validation. + # - "exact": the reference path, plus the fast paths that reproduce its + # rounding exactly. Output is bit-identical to it in the same + # environment; ask for this when a run has to be reproducible or when + # bisecting a numerical bug. + # - "lossless" (default): also mount the fast paths that keep the + # reference math and the precision of every operand and accumulator, + # and only move where the rounding happens. Output matches the + # reference to rounding error rather than bit for bit, by less than the + # same path already moves between two legal environments. + # - "high": also allow model-owned approximate optimizations (lower + # precision, sparse computation, feature caching), which need + # model-specific quality validation. # # It intentionally participates in the dynamic-batch signature. quality: str = "lossless" @@ -604,10 +620,7 @@ class SamplingParams: f"prompt_path must be a txt file, got {self.prompt_path!r}" ) - if self.quality not in QUALITY_LEVELS: - raise ValueError( - f"quality must be one of {list(QUALITY_LEVELS)}, got {self.quality!r}" - ) + self.quality = normalize_quality(self.quality) resolve_skip_softmax_params(self.skip_softmax_params) @@ -1257,16 +1270,17 @@ class SamplingParams: add_argument( "--quality", type=str, - choices=list(QUALITY_LEVELS), + choices=[*QUALITY_LEVELS, *QUALITY_ALIASES], help=( - "Request-level quality: 'lossless' (default) keeps the exact " - "reference path, bit-exact against the reference " - "implementation; 'extra-high' adds only request-gated kernel " - "fusions and does not itself enable sparse or approximate " - "optimization; 'high' includes every extra-high fusion and " - "may also enable " - "model-owned approximate paths. Support and validated " - "deployment constraints are model-specific." + "Request-level quality: 'lossless' (default) runs every fast " + "path that keeps the reference math and operand precision and " + "only moves the rounding, so output matches the reference to " + "rounding error; 'exact' restricts it to the fast paths that " + "reproduce the reference's rounding exactly, giving " + "bit-identical output in the same environment; 'high' may " + "also enable model-owned approximate paths. 'extra-high' is " + "accepted as the former name of 'lossless'. Support and " + "validated deployment constraints are model-specific." ), ) add_argument( diff --git a/python/sglang/multimodal_gen/runtime/managers/gpu_worker.py b/python/sglang/multimodal_gen/runtime/managers/gpu_worker.py index c04c236706..bbf38d7a53 100644 --- a/python/sglang/multimodal_gen/runtime/managers/gpu_worker.py +++ b/python/sglang/multimodal_gen/runtime/managers/gpu_worker.py @@ -102,6 +102,7 @@ from sglang.multimodal_gen.runtime.utils.logging_utils import ( configure_logger, init_logger, ) +from sglang.multimodal_gen.runtime.utils.numerics_policy import apply_numerics_policy from sglang.multimodal_gen.runtime.utils.perf_logger import ( PerformanceLogger, capture_memory_snapshot, @@ -384,6 +385,12 @@ class GPUWorker(GPUWorkerPostTrainingMixin): if not current_platform.is_mps(): current_platform.set_device(current_platform.get_device(self.local_rank)) self._cap_device_memory_for_tests() + apply_numerics_policy( + allow_cudnn_tf32=self.server_args.allow_cudnn_tf32, + allow_bf16_reduced_precision_reduction=( + self.server_args.allow_bf16_reduced_precision_reduction + ), + ) # num_gpus is the total world size across every node; the co-located, # CPU-contending worker count on THIS host is num_gpus // nnodes. local_num_gpus = self.server_args.num_gpus // self.server_args.nnodes diff --git a/python/sglang/multimodal_gen/runtime/models/dits/flux.py b/python/sglang/multimodal_gen/runtime/models/dits/flux.py index da5b71fd62..4a4a6ecf58 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/flux.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/flux.py @@ -175,7 +175,7 @@ def _flux_norm_modulate( Priority: (1) the bit-exact single-kernel LN+modulate -- lossless, so it needs no quality gate and also supersedes the request-gated affine fold wherever it verifies; (2) when the site is mounted - (``quality="extra-high"`` or ``"high"``) and the bit-exact kernel is + (``quality="lossless"`` or ``"high"``) and the bit-exact kernel is unavailable, the modulate folded into the LN affine (one aten kernel; not bit-exact); (3) affine-free LayerNorm + the bit-exact fused modulate. @@ -409,7 +409,7 @@ class FluxGELU(nn.Module): prefix=f"{prefix}.proj" if prefix else "proj", ) self.gelu = nn.GELU(approximate="tanh") - # extra-high/high fusion site: up-proj GEMM + tanh-GELU in the cublasLt + # lossless/high fusion site: up-proj GEMM + tanh-GELU in the cublasLt # epilogue. Off by default; mounted per batch by the denoising stage. mark_fused_gelu_site(self, "proj") @@ -429,7 +429,7 @@ class FluxFusedGELUProj(nn.Module): ``approximate="tanh"`` that keeps the ``net.0.proj`` parameter path. The default path is the bit-exact reference (plain Linear + tanh-GELU); the cublasLt GELU epilogue is mounted per batch by the denoising stage for - requests with ``quality="extra-high"`` or ``quality="high"`` only. + requests with ``quality="lossless"`` or ``quality="high"`` only. """ def __init__(self, proj: nn.Linear): @@ -877,7 +877,7 @@ class FluxSingleTransformerBlock(nn.Module): prefix=f"{prefix}.proj_mlp" if prefix else "proj_mlp", ) self.act_mlp = nn.GELU(approximate="tanh") - # extra-high/high fusion site: proj_mlp GEMM + tanh-GELU in the + # lossless/high fusion site: proj_mlp GEMM + tanh-GELU in the # cublasLt epilogue (mounted per batch by the denoising stage). mark_fused_gelu_site(self, "proj_mlp") proj_out_cls = ( @@ -1044,7 +1044,7 @@ class FluxTransformerBlock(nn.Module): self.norm2 = LayerNorm(dim, eps=1e-6, elementwise_affine=False) self.norm2_context = LayerNorm(dim, eps=1e-6, elementwise_affine=False) - # extra-high/high site: norm2/norm2_context modulate folds into the + # lossless/high site: norm2/norm2_context modulate folds into the # LN affine when mounted. mark_fused_ln_modulate_site(self) @@ -1099,7 +1099,7 @@ class FluxTransformerBlock(nn.Module): activation_fn="gelu-approximate", ) # Re-home each FF's tanh-GELU up-projection onto a marked - # extra-high/high fusion site (bit-exact reference by default). + # lossless/high fusion site (bit-exact reference by default). self.ff.net[0] = FluxFusedGELUProj(self.ff.net[0].proj) self.ff_context.net[0] = FluxFusedGELUProj(self.ff_context.net[0].proj) diff --git a/python/sglang/multimodal_gen/runtime/models/dits/glm_image.py b/python/sglang/multimodal_gen/runtime/models/dits/glm_image.py index 5fd03e68af..8a101dded9 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/glm_image.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/glm_image.py @@ -448,7 +448,7 @@ class GlmImageGELU(nn.Module): quant_config=quant_config, prefix=f"{prefix}.proj" if prefix else "proj", ) - # extra-high/high fusion site: up-proj GEMM + tanh-GELU in cublasLt + # lossless/high fusion site: up-proj GEMM + tanh-GELU in cublasLt # epilogue. Off by default; mounted per batch by the denoising stage. mark_fused_gelu_site(self, "proj") diff --git a/python/sglang/multimodal_gen/runtime/models/dits/ideogram.py b/python/sglang/multimodal_gen/runtime/models/dits/ideogram.py index c9e8332751..63b584ca55 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/ideogram.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/ideogram.py @@ -399,7 +399,7 @@ def _norm_scale( norm: Ideogram4RMSNorm, enable_fused: bool, ) -> torch.Tensor: - """``RMSNorm(x) * (1 + scale)``, fused at extra-high or high quality.""" + """``RMSNorm(x) * (1 + scale)``, fused at high quality.""" if enable_fused: y = fused_rmsnorm_scale( x, @@ -433,7 +433,7 @@ def _gate_residual( norm: Ideogram4RMSNorm, enable_fused: bool, ) -> torch.Tensor: - """``residual + tanh(gate) * RMSNorm(x)``, fused at extra-high or high.""" + """``residual + tanh(gate) * RMSNorm(x)``, fused at high quality.""" if enable_fused: y = fused_rmsnorm_tanh_residual( x, @@ -489,7 +489,7 @@ class Ideogram4TransformerBlock(nn.Module): self.ffn_norm1 = Ideogram4RMSNorm(hidden_size, eps=norm_eps) self.attention_norm2 = Ideogram4RMSNorm(hidden_size, eps=norm_eps) self.ffn_norm2 = Ideogram4RMSNorm(hidden_size, eps=norm_eps) - # extra-high/high fusion sites: each RMSNorm modulate/gate chain + # high-tier fusion sites: each RMSNorm modulate/gate chain # collapses into one Triton kernel (Z-Image bf16-native suite). Off by # default (bit-exact reference path); mounted per batch by the # denoising stage. diff --git a/python/sglang/multimodal_gen/runtime/models/dits/lingbot_video_moe.py b/python/sglang/multimodal_gen/runtime/models/dits/lingbot_video_moe.py index c475e1206e..9edd001e51 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/lingbot_video_moe.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/lingbot_video_moe.py @@ -114,7 +114,7 @@ def _lingbot_gated_residual( Uses the request-gated per-token ``residual_gate_add`` fast path when the block's quality-gated site is enabled; otherwise the reference FP32 - multiply form (bit-exact for ``quality="lossless"``). + multiply form (bit-exact for ``quality="exact"``). """ fused = try_lingbot_video_gated_residual(block, residual, update, gate) if fused is not None: diff --git a/python/sglang/multimodal_gen/runtime/models/dits/longcat_image.py b/python/sglang/multimodal_gen/runtime/models/dits/longcat_image.py index ac6c2e9dbf..9103b600d7 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/longcat_image.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/longcat_image.py @@ -324,7 +324,7 @@ class _LongCatFFN(nn.Module): ] ) self.act = nn.GELU(approximate="tanh") - # extra-high/high site: up-proj GEMM + tanh-GELU epilogue. Off by + # lossless/high site: up-proj GEMM + tanh-GELU epilogue. Off by # default; the denoising stage mounts it per batch. The ModuleDict holds # `proj` in _modules, so getattr resolves it for the fusion helper. mark_fused_gelu_site(self.net[0], "proj") @@ -613,7 +613,7 @@ class _SingleTransformerBlock(nn.Module): prefix=f"{prefix}.proj_mlp", ) self.act_mlp = nn.GELU(approximate="tanh") - # extra-high/high site: proj_mlp GEMM + tanh-GELU epilogue, + # lossless/high site: proj_mlp GEMM + tanh-GELU epilogue, # mounted per batch by the denoising stage; off (bit-exact) by default. mark_fused_gelu_site(self, "proj_mlp") # proj_out: RowParallelLinear reduces sharded [attn | mlp] concat via diff --git a/python/sglang/multimodal_gen/runtime/models/dits/ltx_2.py b/python/sglang/multimodal_gen/runtime/models/dits/ltx_2.py index 3e39022a8e..18ab9df5f7 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/ltx_2.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/ltx_2.py @@ -12,7 +12,7 @@ import torch.nn.functional as F from sglang.kernels.ops.diffusion import ( BitExactFusionGate, - can_use_fused_rmsnorm_scale_shift, + can_use_fused_rmsnorm_modulation, can_use_linear_gelu, can_use_ltx2_qknorm_split_rope_cuda, can_use_modulate_scale_shift_cuda, @@ -204,14 +204,12 @@ def _ltx2_rms_norm_modulate( Folds the weightless RMSNorm and the modulate into one kernel when the request-gated fusion is mounted on ``block`` and the per-call guard - passes; otherwise the verbatim eager reference chain (the ``lossless`` + passes; otherwise the verbatim eager reference chain (the ``exact`` default). The fused kernel is not bit-exact (<=1 bf16 ULP) so it is gated on the request-scoped mount rather than a runtime self-check. """ - if ( - ltx2_rms_norm_modulate_active(block) - and x.is_cuda - and can_use_fused_rmsnorm_scale_shift(x.dtype, x.shape[-1]) + if ltx2_rms_norm_modulate_active(block) and can_use_fused_rmsnorm_modulation( + x, scale, shift ): return fused_ltx2_rms_norm_modulate(x, scale, shift, eps) normed = rms_norm(x, eps) diff --git a/python/sglang/multimodal_gen/runtime/models/dits/qwen_image.py b/python/sglang/multimodal_gen/runtime/models/dits/qwen_image.py index 7b7ca0cc7c..dbd00fbeb5 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/qwen_image.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/qwen_image.py @@ -999,8 +999,8 @@ class QwenImageCrossAttention(nn.Module): prefix=f"{prefix}.to_added_qkv", ) if self._unquantized_added_qkv_is_packed: - # Packing changes BF16 GEMM reduction association. Keep it - # off for lossless and mount it at extra-high or high. + # Packing changes BF16 GEMM reduction association, so it + # is off at exact and mounts at lossless or high. mark_qwen_image_added_qkv_site(self) else: self.add_q_proj = ColumnParallelLinear( diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/flux2_vae_cuda_opt.py b/python/sglang/multimodal_gen/runtime/models/vaes/flux2_vae_cuda_opt.py index 4bf19a53aa..95ee76c0a1 100644 --- a/python/sglang/multimodal_gen/runtime/models/vaes/flux2_vae_cuda_opt.py +++ b/python/sglang/multimodal_gen/runtime/models/vaes/flux2_vae_cuda_opt.py @@ -8,8 +8,8 @@ decoder module family (``ResnetBlock2D`` GroupNorm+SiLU chains, All rewrites are mathematically exact re-associations of the original operators. Wrappers are installed once at VAE load and dispatch on a -decode-scoped :class:`VaeFastPathGate`: ``quality="extra-high"`` and -``quality="high"`` run the fast paths, while the ``"lossless"`` default runs +decode-scoped :class:`VaeFastPathGate`: ``quality="lossless"`` and +``quality="high"`` run the fast paths, while the ``"exact"`` default runs the original module path bit-for-bit. - channels_last: run the decoder in NHWC so cuDNN convs skip the transpose diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/wan_vae_cuda_opt.py b/python/sglang/multimodal_gen/runtime/models/vaes/wan_vae_cuda_opt.py index fc558f1c36..9ebbb22bff 100644 --- a/python/sglang/multimodal_gen/runtime/models/vaes/wan_vae_cuda_opt.py +++ b/python/sglang/multimodal_gen/runtime/models/vaes/wan_vae_cuda_opt.py @@ -4,9 +4,9 @@ Qwen-Image VAE, which is the Wan 2.1 VAE under other class names). Fuses every decoder ``WanRMS_norm -> SiLU`` chain into one Triton kernel on the channels_last_3d layout. Wrappers are installed once at VAE load and -dispatch on a decode-scoped :class:`VaeFastPathGate`: ``quality="extra-high"`` +dispatch on a decode-scoped :class:`VaeFastPathGate`: ``quality="lossless"`` and ``quality="high"`` run the fused kernel (not bitwise-identical to aten, -hence gated). The ``"lossless"`` path preserves aten's normalization reduction +hence gated). The ``"exact"`` path preserves aten's normalization reduction and fuses its FP32 post-ops after first-sight exactness verification. Install is all-or-nothing and fail-closed. """ @@ -160,7 +160,7 @@ class GatedChannelsLastUpsample(nn.Module): # The predicate admits exactly the inputs on which aten itself would # run its NHWC kernel and return a dense channels_last tensor, so the # Triton gather is a layout- and value-identical replacement: it runs - # on the lossless path too (the gate only controls the stride + # on the exact path too (the gate only controls the stride # canonicalisation above). if up.size is None and can_use_nearest_upsample_nhwc( x, up.scale_factor, up.mode diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/decoding.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/decoding.py index b0e5df668b..8410a2d3dd 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/decoding.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/decoding.py @@ -11,7 +11,7 @@ import torch import torch.nn as nn from sglang.multimodal_gen.configs.sample.sampling_params import ( - quality_allows_kernel_fusions, + quality_allows, ) from sglang.multimodal_gen.runtime.distributed import ( get_decode_parallel_world_size, @@ -334,9 +334,11 @@ class DecodingStage(PipelineStage): assert vae is not None self.vae = vae + # The decoder fast paths re-associate the reference operators + # without lowering any precision, so they belong to "lossless". with use_vae_fast_path( vae, - quality_allows_kernel_fusions(batch.sampling_params.quality), + quality_allows(batch.sampling_params.quality, "lossless"), ): frames = self.decode(batch.latents, server_args, vae_dtype=vae_dtype) diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/denoising.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/denoising.py index 5a83786c42..aa16ec24ac 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/denoising.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/denoising.py @@ -56,7 +56,7 @@ from sglang.multimodal_gen.configs.pipeline_configs.flux import ( ) from sglang.multimodal_gen.configs.pipeline_configs.zimage import ZImagePipelineConfig from sglang.multimodal_gen.configs.sample.sampling_params import ( - quality_allows_kernel_fusions, + quality_allows, resolve_skip_softmax_params, ) from sglang.multimodal_gen.configs.task_type import get_request_task_type @@ -177,70 +177,95 @@ from sglang.multimodal_gen.runtime.utils.torch_compile import ( logger = init_logger(__name__) +# Request-gated DiT fusions and the lowest quality level that may mount each +# one. A fusion belongs to "lossless" when it keeps the reference math and the +# precision of every operand and accumulator, changing only where or in which +# order the rounding happens; one that lowers a precision or quantizes belongs +# to "high". No entry is tier "exact": the default must keep running the same +# kernels it runs today, so a by-construction bit-exact fast path has to come +# with a BitExactFusionGate before it can move there. _QUALITY_FUSION_HANDLERS: tuple[ - tuple[str, Callable[[nn.Module], bool], Callable[[nn.Module], None]], ... + tuple[str, str, Callable[[nn.Module], bool], Callable[[nn.Module], None]], ... ] = ( ( + # quantizes FC2's input before the reference BF16 intermediate exists + "high", "FLUX.2 NVFP4 FC1+SwiGLU+quant", mount_flux2_nvfp4_swiglu_quant, unmount_flux2_nvfp4_swiglu_quant, ), ( + "lossless", "fused linear+GELU (cublasLt epilogue)", mount_fused_linear_gelu, unmount_fused_linear_gelu, ), ( + "lossless", "Wan NVFP4 fused bias+GELU", mount_nvfp4_bias_gelu, unmount_nvfp4_bias_gelu, ), ( + "lossless", "Qwen-Image fused added-QKV", mount_qwen_image_added_qkv, unmount_qwen_image_added_qkv, ), ( + "lossless", "fused LN+modulate (affine folding)", mount_fused_ln_modulate, unmount_fused_ln_modulate, ), ( + "lossless", "LTX-2 Hopper QKNorm+split-RoPE", mount_ltx2_qknorm_split_rope, unmount_ltx2_qknorm_split_rope, ), ( + "lossless", "LTX-2 fused RMSNorm+modulate", mount_ltx2_rms_norm_modulate, unmount_ltx2_rms_norm_modulate, ), ( + # Ideogram's reference keeps the norm statistics in fp32; the + # BF16-native kernel rounds them, which lowers accumulation precision + "high", "fused gate RMSNorm (BF16-native Triton)", mount_fused_gate_rmsnorm, unmount_fused_gate_rmsnorm, ), ( + "lossless", "HunyuanVideo strided QK RMSNorm", mount_hunyuan_qknorm, unmount_hunyuan_qknorm, ), ( + "lossless", "LingBot Video fused RMSNorm", mount_lingbot_video_rmsnorm, unmount_lingbot_video_rmsnorm, ), ( + "lossless", "LingBot Video per-token gated residual", mount_lingbot_video_gated_residual, unmount_lingbot_video_gated_residual, ), ( + "lossless", "Helios per-token gated residual", mount_helios_gated_residual, unmount_helios_gated_residual, ), ( + # the first attention GEMM takes BF16 inputs where the reference + # promotes them to FP32 + "high", "SANA-Video BF16-input linear attention", mount_sana_video_linear_attention, unmount_sana_video_linear_attention, @@ -359,11 +384,12 @@ class DenoisingStage(PipelineStage, RolloutDenoisingMixin): self._cache_dit_request_overrides: dict[str, Any] = {} # Overrides key the mounted hooks were built from; None when unmounted. self._cache_dit_active_key: tuple | None = None - # Whether request-scoped extra-high-or-higher fusions are mounted. - self._quality_fusions_mounted = False + # The quality level whose request-scoped fusions are mounted. + self._mounted_quality = "exact" self._torch_compile_registry = CompiledModuleRegistry() - # Breakable CUDA graph runners, one per transformer module (lazy). - self._bcg_runners: dict[int, Any] = {} + # Breakable CUDA graph runners, lazily created per (module, quality + # level); see _maybe_get_bcg_runner for why the level is in the key. + self._bcg_runners: dict[tuple[int, str], Any] = {} hidden_size = self.server_args.pipeline_config.dit_config.hidden_size num_attention_heads = ( @@ -763,40 +789,32 @@ class DenoisingStage(PipelineStage, RolloutDenoisingMixin): def _maybe_toggle_quality_fusions(self, batch: Req) -> None: """Mount/unmount request-gated kernel fusions for this batch. - These fusions are numerically equivalent only at half-precision - rounding level (not bit-exact), so they are mounted for both - ``quality="extra-high"`` and ``quality="high"``. The ``"lossless"`` - default runs the reference path bit-for-bit. ``quality`` participates - in the dynamic-batch signature, making this transition safe at the - batch boundary. Mounting is all-or-nothing per transformer and fusion + Each fusion declares the lowest quality level that may mount it (see + ``_QUALITY_FUSION_HANDLERS``), so a request mounts the fusions of its + own tier and of every stricter one; ``"exact"`` mounts none of them + and runs the reference path bit-for-bit. ``quality`` participates in + the dynamic-batch signature, making this transition safe at the batch + boundary. Mounting is all-or-nothing per transformer and fusion family; models without marked sites are no-ops. + + Under breakable CUDA graphs the mounted set is baked into whatever was + captured, which is why the runner is keyed by level as well as module + (see :meth:`_maybe_get_bcg_runner`): changing level here cannot + silently replay another level's kernels. """ quality = getattr(batch.sampling_params, "quality", "lossless") - want = quality_allows_kernel_fusions(quality) - if want == self._quality_fusions_mounted: + if quality == self._mounted_quality: return mounted_fusions: set[str] = set() for transformer in filter(None, [self.transformer, self.transformer_2]): - for description, mount, unmount in _QUALITY_FUSION_HANDLERS: - if want: + for tier, description, mount, unmount in _QUALITY_FUSION_HANDLERS: + if quality_allows(quality, tier): if mount(transformer): mounted_fusions.add(description) else: unmount(transformer) - if want and mounted_fusions and self.server_args.enable_breakable_cuda_graph: - for transformer in filter(None, [self.transformer, self.transformer_2]): - for _, _, unmount in _QUALITY_FUSION_HANDLERS: - unmount(transformer) - descriptions = ", ".join(sorted(mounted_fusions)) - raise ValueError( - f"quality={quality!r} cannot be used with breakable CUDA graphs for " - f"this model because its request-scoped DiT fusions " - f"({descriptions}) do not match the lossless warmup graphs. " - "Disable breakable CUDA graphs or use quality='lossless'." - ) - - self._quality_fusions_mounted = want + self._mounted_quality = quality for description in sorted(mounted_fusions): logger.debug("Mounted %s for quality=%s", description, quality) @@ -2589,19 +2607,38 @@ class DenoisingStage(PipelineStage, RolloutDenoisingMixin): return None if not isinstance(current_model, nn.Module): return None - key = id(current_model) + # A captured graph bakes in whichever fusions were mounted when it was + # recorded, and the capture signature only covers the tensors -- so a + # runner is per (module, quality level). Warmup runs at the default + # level and captures its graphs there; a request at another level + # finds an empty runner and the eager fallback runs the fusion set it + # actually asked for. Sharing one runner would replay the warmup + # level's kernels under a different level's name. + key = (id(current_model), self._mounted_quality) runner = self._bcg_runners.get(key) if runner is None: from sglang.multimodal_gen.runtime.breakable_cuda_graph.runner import ( DiffusionBreakableCudaGraphRunner, ) + # Another level already has a runner, so this one arrived after + # warmup and will never hold a captured graph. + served_another_level = any( + model_id == id(current_model) for model_id, _ in self._bcg_runners + ) # DenoisingStage can switch between transformer and transformer_2; # each module owns separate graph state and static input buffers. runner = DiffusionBreakableCudaGraphRunner( current_model, get_local_torch_device() ) self._bcg_runners[key] = runner + if served_another_level: + logger.info_once( + "quality=%s has no captured breakable CUDA graphs (warmup " + "captures the server's default level), so its requests run " + "eager.", + self._mounted_quality, + ) return runner def prepare_sta_param(self, batch: Req, server_args: ServerArgs): diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/helios_denoising.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/helios_denoising.py index f046034e1f..aa10545429 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/helios_denoising.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/helios_denoising.py @@ -18,7 +18,7 @@ from sglang.kernels.ops.diffusion import ( unmount_helios_gated_residual, ) from sglang.multimodal_gen.configs.sample.sampling_params import ( - quality_allows_kernel_fusions, + quality_allows, ) from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context @@ -116,11 +116,12 @@ class HeliosChunkedDenoisingStage(PipelineStage): self._quality_fusions_mounted = False def _maybe_toggle_quality_fusions(self, batch: Req) -> None: - # Mount the request-scoped Helios gated-residual fast path for - # quality="extra-high"/"high"; "lossless" keeps the reference - # FP32-multiply form bit-for-bit. + # The gated-residual fusion rounds the FP32 gate and update to BF16 + # before multiplying, which moves a rounding without lowering the + # reference's own operand precision: tier "lossless". The "exact" + # default keeps the reference FP32-multiply form bit-for-bit. quality = getattr(batch.sampling_params, "quality", "lossless") - want = quality_allows_kernel_fusions(quality) + want = quality_allows(quality, "lossless") if want == self._quality_fusions_mounted: return self._quality_fusions_mounted = want diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/constants.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/constants.py index d7293e5f58..2108a3762e 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/constants.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/constants.py @@ -57,8 +57,8 @@ MINIMAX_H3_DEFAULT_BRANCHES: tuple = ({"name": "cond_1"},) # Audited 4xH200 T2VA Cache-DiT parameters for quality="high": # (warmup steps, residual-difference threshold, max consecutive cached steps). -# Measured SSIM 0.931 / PSNR 28.16 dB against quality="lossless" on the -# validated workload. quality="lossless" (the default) uses no Cache-DiT +# Measured SSIM 0.931 / PSNR 28.16 dB against quality="exact" on the +# validated workload. quality="exact" (the default) uses no Cache-DiT # configuration at all. Process-wide SGLANG_CACHE_DIT_* environment controls # remain available for manual experiments and are independent of this field. MINIMAX_H3_HIGH_QUALITY_CACHE_DIT_CONFIG: tuple[int, float, int] = (4, 0.04, 1) diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/denoising.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/denoising.py index 690645e007..544e8e8c90 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/denoising.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/denoising.py @@ -455,7 +455,7 @@ class MiniMaxH3DenoisingStage(DenoisingStage): scheduler=None, pipeline=pipeline, ) - self._minimax_h3_quality = "lossless" + self._minimax_h3_quality = "exact" self._minimax_h3_cache_mode: str | None = None def _owns_compile_warmup_lifecycle(self) -> bool: @@ -463,7 +463,7 @@ class MiniMaxH3DenoisingStage(DenoisingStage): def _cache_dit_requested(self) -> bool: return ( - getattr(self, "_minimax_h3_quality", "lossless") == "high" + getattr(self, "_minimax_h3_quality", "exact") == "high" or super()._cache_dit_requested() ) @@ -592,7 +592,7 @@ class MiniMaxH3DenoisingStage(DenoisingStage): def _cache_dit_scm_masks( self, primary_num_steps: int, secondary_num_steps: int | None = None ) -> tuple[str, str, list[int] | None, list[int] | None]: - if getattr(self, "_minimax_h3_quality", "lossless") == "high": + if getattr(self, "_minimax_h3_quality", "exact") == "high": return "none", "dynamic", None, None return super()._cache_dit_scm_masks(primary_num_steps, secondary_num_steps) @@ -604,7 +604,7 @@ class MiniMaxH3DenoisingStage(DenoisingStage): *, secondary: bool = False, ) -> CacheDitConfig: - if secondary or getattr(self, "_minimax_h3_quality", "lossless") != "high": + if secondary or getattr(self, "_minimax_h3_quality", "exact") != "high": return super()._build_cache_dit_config( num_inference_steps, steps_computation_mask, diff --git a/python/sglang/multimodal_gen/runtime/platforms/cuda.py b/python/sglang/multimodal_gen/runtime/platforms/cuda.py index 19388b1b9f..9005d86995 100644 --- a/python/sglang/multimodal_gen/runtime/platforms/cuda.py +++ b/python/sglang/multimodal_gen/runtime/platforms/cuda.py @@ -841,8 +841,8 @@ class CudaPlatformBase(Platform): """Install the quality-gated FLUX.2 / AutoencoderKL / Wan / Qwen-Image VAE decoder fast paths. - Requests with quality="extra-high" or "high" run the fast paths; the - "lossless" default runs the original module path bit-for-bit. See + Requests with quality="lossless" or "high" run the fast paths; the + "exact" default runs the original module path bit-for-bit. See flux2_vae_cuda_opt and wan_vae_cuda_opt for details. """ try: diff --git a/python/sglang/multimodal_gen/runtime/server_args/server_args.py b/python/sglang/multimodal_gen/runtime/server_args/server_args.py index ac2e2a2870..df8bc672bc 100644 --- a/python/sglang/multimodal_gen/runtime/server_args/server_args.py +++ b/python/sglang/multimodal_gen/runtime/server_args/server_args.py @@ -483,6 +483,12 @@ class ServerArgs(DisaggServerArgsMixin): # stay eager). Mutually exclusive with --enable-torch-compile and # Cache-DiT; BCG takes priority when more than one is requested. # + # PyTorch's two approximate numerics defaults, set explicitly per worker + # (see runtime/utils/numerics_policy.py). They are process-global, so they + # cannot follow a request's quality level; the defaults keep PyTorch's own + # values so the exact tier's speed is unchanged. + allow_cudnn_tf32: bool = True + allow_bf16_reduced_precision_reduction: bool = True # BCG graphs are resolution-specific, so --warmup-resolutions is required # when BCG is enabled: every requested resolution is captured at warmup so # serving never triggers a fresh capture. @@ -2445,6 +2451,23 @@ class ServerArgs(DisaggServerArgsMixin): default=ServerArgs.offload_during_compile, help="Offload components during the torch.compile warmup (the DiT layerwise) so max-autotune fits on tighter-memory GPUs, then restore the configured residency for serving. Skipped when the DiT is already layerwise-offloaded, or under cache-dit / FSDP.", ) + parser.add_argument( + "--allow-cudnn-tf32", + action=StoreBoolean, + default=ServerArgs.allow_cudnn_tf32, + help="Let cuDNN run fp32 convolutions on TF32 tensor cores, which " + "truncates their inputs to 10 mantissa bits (PyTorch's own " + "default). Affects the fp32 VAE decoders; pass false together " + "with --allow-bf16-reduced-precision-reduction false for " + "reference-precision fp32 and bf16 math.", + ) + parser.add_argument( + "--allow-bf16-reduced-precision-reduction", + action=StoreBoolean, + default=ServerArgs.allow_bf16_reduced_precision_reduction, + help="Let a bf16 GEMM accumulate its split-K partials below fp32 " + "(PyTorch's own default). See --allow-cudnn-tf32.", + ) parser.add_argument( "--enable-breakable-cuda-graph", action=StoreBoolean, diff --git a/python/sglang/multimodal_gen/runtime/utils/numerics_policy.py b/python/sglang/multimodal_gen/runtime/utils/numerics_policy.py new file mode 100644 index 0000000000..255b4359ad --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/utils/numerics_policy.py @@ -0,0 +1,65 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Explicit control of the two approximate numerics PyTorch enables by default. + +``torch.backends.cudnn.allow_tf32`` truncates fp32 convolution inputs to 10 +mantissa bits, and +``torch.backends.cuda.matmul.allow_bf16_reduced_precision_reduction`` lets a +bf16 GEMM accumulate split-K partials below fp32. Both change what is +computed, so both sit at the ``high`` tier of ``QUALITY_LEVELS`` while the +runtime serves ``exact`` and ``lossless`` requests through the same process. + +They cannot follow the request: they are process-global and a server mixes +tiers across concurrent batches. The runtime therefore sets them once per +worker from the server args and states the result, so a tier's numerical +contract is auditable instead of inherited from whichever PyTorch build is +installed. + +The defaults keep PyTorch's own values. Turning TF32 off makes fp32 +convolutions -- the VAE decoders that default to ``vae_precision="fp32"`` -- +run on fp32 CUDA cores rather than tensor cores, so flipping the default would +slow the exact tier down. A deployment that needs the strict contract sets +both to false and pays that cost knowingly. +""" + +from __future__ import annotations + +import torch + +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +logger = init_logger(__name__) + + +def apply_numerics_policy( + *, allow_cudnn_tf32: bool, allow_bf16_reduced_precision_reduction: bool +) -> None: + """Set both approximate-numerics switches and log what this worker runs.""" + torch.backends.cudnn.allow_tf32 = allow_cudnn_tf32 + torch.backends.cuda.matmul.allow_bf16_reduced_precision_reduction = ( + allow_bf16_reduced_precision_reduction + ) + approximate = [ + name + for name, enabled in ( + ("cudnn TF32", allow_cudnn_tf32), + ( + "bf16 reduced-precision reduction", + allow_bf16_reduced_precision_reduction, + ), + ) + if enabled + ] + if approximate: + logger.info_once( + "Numerics policy: %s enabled. These lower precision below the " + "reference, so quality='exact' is bit-reproducible on this server " + "but not free of approximate library defaults. Pass " + "--allow-cudnn-tf32 false --allow-bf16-reduced-precision-reduction " + "false for the strict contract (slower fp32 convolutions).", + " and ".join(approximate), + ) + else: + logger.info_once( + "Numerics policy: cudnn TF32 and bf16 reduced-precision reduction " + "are both disabled; fp32 and bf16 math keeps reference precision." + ) diff --git a/python/sglang/multimodal_gen/test/quality_tier_admission.py b/python/sglang/multimodal_gen/test/quality_tier_admission.py new file mode 100644 index 0000000000..6502119de9 --- /dev/null +++ b/python/sglang/multimodal_gen/test/quality_tier_admission.py @@ -0,0 +1,206 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Admission checks for a fast path that claims the ``lossless`` tier. + +A ``lossless``-tier fast path keeps the reference math and the precision of +every operand and accumulator; only the place or the order of the rounding +moves. That claim is not provable from a single tolerance, so these helpers +implement the checks a fusion has to pass before its tier is believed, and +replace the per-case thresholds each fusion test used to invent: + +* :func:`assert_error_no_worse_than_reference` -- the operator-level gate. Both + the current implementation and the candidate are scored against an fp64 + evaluation of the same math; the candidate may not be more than + ``max_error_ratio`` times worse on RMS or max error, and its error may not + carry a systematic sign. +* :func:`assert_special_values_match` -- NaN, Inf, fully-masked rows and + non-contiguous inputs keep their current behavior. +* :func:`discrete_flip_rate` -- when a quantizer, top-k or threshold reads the + output, the quantity that matters is how often the discrete result changes, + not the size of the continuous error. + +Two gates are deliberately not automated here, because they are properties of +the change rather than of one tensor comparison: + +* the static conditions (accumulation at fp32 or better, no operand or + intermediate precision lowered, an approximate transcendental within 0.1 ulp + of the output dtype, stability no worse than the reference: two-pass or + Welford variance, max-subtracted softmax). A fast path that breaks one of + them is approximate by construction, and no measurement can admit it. +* the end-to-end drift band -- the difference against the exact tier has to sit + inside the spread the same reference path already shows across two legal + environments (H100 against H200, TP1 against TP2, two cuBLAS versions). That + needs a server run, so it belongs to the model's consistency suite. + +Feed the operator-level gate activations captured from the model, not only +random tensors: DiT activations carry outlier channels that decide whether a +reassociated reduction stays within the reference's error. +""" + +from __future__ import annotations + +from collections.abc import Callable, Sequence +from dataclasses import dataclass + +import torch + +Impl = Callable[..., torch.Tensor] + + +@dataclass(frozen=True) +class ErrorScore: + """One implementation's error against the fp64 evaluation of its math.""" + + rms: float + max_abs: float + mean: float + """Signed mean error; a value far from 0 means the rounding is biased.""" + + def __str__(self) -> str: + return f"rms={self.rms:.3e} max={self.max_abs:.3e} mean={self.mean:+.3e}" + + +def score_against(reference_fp64: torch.Tensor, actual: torch.Tensor) -> ErrorScore: + """Score ``actual`` against an fp64 evaluation of the same math.""" + if reference_fp64.shape != actual.shape: + raise ValueError( + f"shape mismatch: reference {tuple(reference_fp64.shape)} vs " + f"actual {tuple(actual.shape)}" + ) + error = actual.to(torch.float64) - reference_fp64 + return ErrorScore( + rms=float(error.pow(2).mean().sqrt()), + max_abs=float(error.abs().max()), + mean=float(error.mean()), + ) + + +def assert_error_no_worse_than_reference( + *, + reference_fp64: torch.Tensor, + baseline: torch.Tensor, + candidate: torch.Tensor, + max_error_ratio: float = 1.5, + max_bias_in_rms: float = 0.25, + label: str = "candidate", +) -> tuple[ErrorScore, ErrorScore]: + """Admit ``candidate`` to the lossless tier on operator-level error. + + ``baseline`` is what the model runs today and ``reference_fp64`` the same + math evaluated in fp64. The candidate passes when it is no more than + ``max_error_ratio`` times the baseline's RMS and max error -- the baseline + sets the scale, because the reference path is itself only one sample of + the rounding -- and when its mean error stays within ``max_bias_in_rms`` + of its own RMS, which catches a rounding that always leans one way and + would accumulate over layers and steps. + """ + base = score_against(reference_fp64, baseline) + cand = score_against(reference_fp64, candidate) + + for metric, base_value, cand_value in ( + ("RMS", base.rms, cand.rms), + ("max", base.max_abs, cand.max_abs), + ): + # An exactly-zero baseline error means the reference math is + # representable in the output dtype; hold the candidate to the same. + budget = base_value * max_error_ratio + if cand_value > budget: + raise AssertionError( + f"{label} is not lossless-tier: {metric} error {cand_value:.3e} " + f"exceeds {max_error_ratio}x the reference path's " + f"{base_value:.3e}. reference={base} candidate={cand}" + ) + if cand.rms > 0 and abs(cand.mean) > max_bias_in_rms * cand.rms: + raise AssertionError( + f"{label} has a biased rounding: mean error {cand.mean:+.3e} is " + f"more than {max_bias_in_rms} of its RMS {cand.rms:.3e}, so the " + f"error accumulates instead of cancelling. candidate={cand}" + ) + return base, cand + + +def assert_special_values_match( + *, + baseline: Impl, + candidate: Impl, + cases: dict[str, Sequence[torch.Tensor]], + label: str = "candidate", +) -> None: + """Both paths agree on NaN, Inf, empty and non-contiguous inputs. + + Agreement is structural: the same NaN positions, the same infinities with + the same signs, finite where the reference is finite, or the same + exception type. A fast path that swallows a NaN (``fmaxf``), returns zeros + where the reference returns NaN, or lets a NaN leak into rows it should + not reach is a defect rather than a tier. + + A kernel that ignores a stride or aliases its inputs reads the wrong + elements, which is a magnitude error rather than a structural one: pass a + non-contiguous input to :func:`assert_error_no_worse_than_reference` as + well, where it shows up as an error far outside the budget. + + It deliberately does not compare the magnitude of the finite values. A + lossless-tier path differs from the reference there by construction, and + judging that difference is :func:`assert_error_no_worse_than_reference`'s + job, on real activations rather than on these edge-case inputs. An earlier + version compared them with ``torch.allclose`` defaults, which rejected + every bf16 fast path this gate exists to admit. + """ + for name, inputs in cases.items(): + try: + expected = baseline(*inputs) + except Exception as exc: # noqa: BLE001 - the reference's own behavior + expected_error: type[BaseException] | None = type(exc) + expected = None + else: + expected_error = None + + try: + actual = candidate(*inputs) + except Exception as exc: # noqa: BLE001 - compared against the reference + if expected_error is None: + raise AssertionError( + f"{label} raised {type(exc).__name__} on {name!r} where " + f"the reference path returned a tensor" + ) from exc + if type(exc) is not expected_error: + raise AssertionError( + f"{label} raised {type(exc).__name__} on {name!r}, the " + f"reference path raised {expected_error.__name__}" + ) from exc + continue + + if expected_error is not None: + raise AssertionError( + f"{label} returned a tensor on {name!r} where the reference " + f"path raised {expected_error.__name__}" + ) + if actual.shape != expected.shape: + raise AssertionError( + f"{label} returns {tuple(actual.shape)} on {name!r} where the " + f"reference path returns {tuple(expected.shape)}" + ) + if not torch.equal(actual.isnan(), expected.isnan()): + raise AssertionError(f"{label} moves NaNs on {name!r}") + # Signed, so an inf that flips direction is caught too. + for sign, what in ((1, "+Inf"), (-1, "-Inf")): + if not torch.equal( + actual == sign * torch.inf, expected == sign * torch.inf + ): + raise AssertionError(f"{label} moves {what}s on {name!r}") + + +def discrete_flip_rate(baseline: torch.Tensor, candidate: torch.Tensor) -> float: + """Fraction of entries whose discrete downstream result differs. + + Pass the discrete values -- quantized codes, top-k indices, threshold + decisions -- not the continuous tensors they came from. One ulp upstream of + a quantizer or a ``skip-softmax`` threshold is no longer a rounding + difference, so a lossless-tier claim in front of one is judged on this rate. + """ + if baseline.shape != candidate.shape: + raise ValueError( + f"shape mismatch: {tuple(baseline.shape)} vs {tuple(candidate.shape)}" + ) + if baseline.numel() == 0: + return 0.0 + return float((baseline != candidate).to(torch.float64).mean()) diff --git a/python/sglang/multimodal_gen/test/server/gpu_cases.py b/python/sglang/multimodal_gen/test/server/gpu_cases.py index 83c62dcd36..8746bdfcbe 100644 --- a/python/sglang/multimodal_gen/test/server/gpu_cases.py +++ b/python/sglang/multimodal_gen/test/server/gpu_cases.py @@ -1203,6 +1203,8 @@ TWO_GPU_CASES = [ ulysses_degree=1, ring_degree=2, ), + # Keeps the pre-rename spelling of "lossless" so the compatibility + # alias is covered end to end; the case id is also a perf-baseline key. replace(T2I_sampling_params, extras={"quality": "extra-high"}), run_component_accuracy_check=False, run_models_api_check=False, diff --git a/python/sglang/multimodal_gen/test/server/testcase_configs.py b/python/sglang/multimodal_gen/test/server/testcase_configs.py index 4d1b7d2a93..979254c214 100644 --- a/python/sglang/multimodal_gen/test/server/testcase_configs.py +++ b/python/sglang/multimodal_gen/test/server/testcase_configs.py @@ -361,6 +361,27 @@ class DiffusionTestCase: ): raise ValueError(f"{self.id}: request warmup requires non-realtime metrics") + # A consistency golden records one path, and these were recorded on the + # reference one, so the case asks for it by name instead of inheriting + # whichever level is the server default. Without this the check would + # answer two questions at once -- "did the code regress" and "how far + # is the default level from the goldens" -- and spend its whole budget + # on the second: the default's own drift already sits at SSIM 0.91 + # against goldens whose threshold is 0.92. A case that names a level + # keeps it; refreshing the goldens onto the default level is what + # removes the pin. + # Replaces rather than mutates: several cases share one module-level + # sampling-params instance. + if self.run_consistency_check and "quality" not in self.sampling_params.extras: + object.__setattr__( + self, + "sampling_params", + replace( + self.sampling_params, + extras={**self.sampling_params.extras, "quality": "exact"}, + ), + ) + has_startup_lora = self.server_args.lora_path is not None has_dynamic_lora = self.server_args.dynamic_lora_path is not None has_second_lora = self.server_args.second_lora_path is not None diff --git a/python/sglang/multimodal_gen/test/unit/test_diffusion_bcg_padding.py b/python/sglang/multimodal_gen/test/unit/test_diffusion_bcg_padding.py index 784beb4b10..e0d051ca88 100644 --- a/python/sglang/multimodal_gen/test/unit/test_diffusion_bcg_padding.py +++ b/python/sglang/multimodal_gen/test/unit/test_diffusion_bcg_padding.py @@ -68,55 +68,127 @@ class TestQualityFusionBCGCompatibility(unittest.TestCase): self.stage.server_args = SimpleNamespace(enable_breakable_cuda_graph=True) self.stage.transformer = OtherTransformer2DModel() self.stage.transformer_2 = None - self.stage._quality_fusions_mounted = False + self.stage._mounted_quality = "exact" @staticmethod def _batch(quality: str): return SimpleNamespace(sampling_params=SimpleNamespace(quality=quality)) - def test_rejects_fusion_levels_when_they_would_replace_captured_graph(self): - for quality in ("extra-high", "high"): - with self.subTest(quality=quality): - unmounted = [] - handlers = ( - ( - "test fusion", - lambda _: True, - lambda transformer: unmounted.append(transformer), - ), - ) + def test_each_quality_level_gets_its_own_bcg_runner(self): + # A captured graph bakes in the fusions mounted when it was recorded, + # and the capture signature only covers the tensors. Warmup captures + # at the server's default level, so a request at another level must + # not reach those graphs: it gets an empty runner, and the eager + # fallback runs the fusion set it actually asked for. + self.stage._bcg_runners = {} + model = self.stage.transformer + built = [] - with patch.object( - denoising_module, "_QUALITY_FUSION_HANDLERS", handlers - ): - with self.assertRaisesRegex(ValueError, "lossless warmup graphs"): - self.stage._maybe_toggle_quality_fusions(self._batch(quality)) + class _FakeRunner: + def __init__(self, transformer, device): + built.append(transformer) - self.assertEqual(unmounted, [self.stage.transformer]) - self.assertFalse(self.stage._quality_fusions_mounted) + with ( + patch.object( + denoising_module, "get_local_torch_device", return_value="cpu" + ), + patch( + "sglang.multimodal_gen.runtime.breakable_cuda_graph.runner." + "DiffusionBreakableCudaGraphRunner", + _FakeRunner, + ), + ): + self.stage._mounted_quality = "lossless" + default_runner = self.stage._maybe_get_bcg_runner(model) + self.assertIs(self.stage._maybe_get_bcg_runner(model), default_runner) - def test_allows_high_when_model_has_no_dit_quality_fusions(self): - handlers = (("test fusion", lambda _: False, lambda _: None),) + self.stage._mounted_quality = "exact" + exact_runner = self.stage._maybe_get_bcg_runner(model) - with patch.object(denoising_module, "_QUALITY_FUSION_HANDLERS", handlers): - self.stage._maybe_toggle_quality_fusions(self._batch("high")) + self.assertIsNot(exact_runner, default_runner) + self.assertEqual(len(built), 2) - self.assertTrue(self.stage._quality_fusions_mounted) - - def test_high_keeps_extra_high_fusions_mounted(self): - self.stage.server_args.enable_breakable_cuda_graph = False + def test_mounting_a_fusion_under_bcg_is_allowed(self): + # This used to raise: with one runner per module, switching level + # would have replayed the warmup level's kernels under another + # level's name. The runner key carries the level now, so the request + # simply runs. mounted = [] handlers = ( - ("test fusion", lambda _: mounted.append(True) or True, lambda _: None), + ( + "lossless", + "test fusion", + lambda t: mounted.append(t) or True, + lambda _: None, + ), ) with patch.object(denoising_module, "_QUALITY_FUSION_HANDLERS", handlers): - self.stage._maybe_toggle_quality_fusions(self._batch("extra-high")) - self.assertTrue(self.stage._quality_fusions_mounted) + self.stage._maybe_toggle_quality_fusions(self._batch("lossless")) + + self.assertEqual(mounted, [self.stage.transformer]) + self.assertEqual(self.stage._mounted_quality, "lossless") + + def test_allows_high_when_model_has_no_dit_quality_fusions(self): + handlers = (("lossless", "test fusion", lambda _: False, lambda _: None),) + + with patch.object(denoising_module, "_QUALITY_FUSION_HANDLERS", handlers): self.stage._maybe_toggle_quality_fusions(self._batch("high")) - self.assertEqual(mounted, [True]) - self.assertTrue(self.stage._quality_fusions_mounted) + self.assertEqual(self.stage._mounted_quality, "high") + + def test_an_exact_request_unmounts_what_a_faster_tier_left_behind(self): + self.stage.server_args.enable_breakable_cuda_graph = False + mounted, unmounted = [], [] + handlers = ( + ( + "lossless", + "test fusion", + lambda t: mounted.append(t) or True, + lambda t: unmounted.append(t), + ), + ) + + with patch.object(denoising_module, "_QUALITY_FUSION_HANDLERS", handlers): + self.stage._maybe_toggle_quality_fusions(self._batch("lossless")) + self.assertEqual(mounted, [self.stage.transformer]) + + # Back to the default at the next batch boundary: the fusion has + # to come off, or an exact request silently keeps running it. + self.stage._maybe_toggle_quality_fusions(self._batch("exact")) + + self.assertEqual(unmounted, [self.stage.transformer]) + self.assertEqual(self.stage._mounted_quality, "exact") + + def test_a_fusion_runs_only_from_the_tier_that_declares_it(self): + self.stage.server_args.enable_breakable_cuda_graph = False + mounted = [] + handlers = ( + ( + "lossless", + "equivalent fusion", + lambda _: mounted.append("equivalent") or True, + lambda _: None, + ), + ( + "high", + "approximate fusion", + lambda _: mounted.append("approximate") or True, + lambda _: None, + ), + ) + + with patch.object(denoising_module, "_QUALITY_FUSION_HANDLERS", handlers): + self.stage._maybe_toggle_quality_fusions(self._batch("lossless")) + self.assertEqual(mounted, ["equivalent"]) + self.assertEqual(self.stage._mounted_quality, "lossless") + + mounted.clear() + self.stage._maybe_toggle_quality_fusions(self._batch("high")) + + # "high" is cumulative: it keeps the equivalent fusion and adds its own + self.assertEqual(mounted, ["equivalent", "approximate"]) + self.assertEqual(self.stage._mounted_quality, "high") def _fake_cache_dit_batch(*, is_warmup: bool) -> SimpleNamespace: @@ -645,6 +717,8 @@ class TestDiffusionBCGPadding(unittest.TestCase): def test_bcg_runner_cache_is_per_model_module(self): self.stage.server_args = SimpleNamespace(enable_breakable_cuda_graph=True) self.stage._bcg_runners = {} + # The cache key is (module, quality level); this one varies the module. + self.stage._mounted_quality = "lossless" def fake_runner(model, device): return SimpleNamespace(model=model, device=device) diff --git a/python/sglang/multimodal_gen/test/unit/test_ltx2_rmsnorm_modulate_fallback.py b/python/sglang/multimodal_gen/test/unit/test_ltx2_rmsnorm_modulate_fallback.py new file mode 100644 index 0000000000..c2fa083e5f --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_ltx2_rmsnorm_modulate_fallback.py @@ -0,0 +1,84 @@ +# SPDX-License-Identifier: Apache-2.0 +"""A mounted fast path falls back on inputs it cannot serve. + +The fused RMSNorm+modulate launcher raises on anything outside its contract, +which is right for a caller that has already committed. The LTX-2 site has +not committed: it is request-gated, so an input it cannot serve has to go +down the reference chain instead of failing the request. + +It did fail one. The site guarded on dtype and width only, while the launcher +also requires `scale` and `shift` to be contiguous `[B, 1, D]` rows, and +LTX-2.5's are not -- so every denoising step raised +"scale and shift must be contiguous [B, 1, D] tensors" as soon as the fusion +was mounted. Nothing caught it earlier because the fusion only mounted above +the default level. +""" + +import sys +import unittest + +import pytest +import torch + +from sglang.kernels.ops.diffusion import can_use_fused_rmsnorm_modulation + + +@unittest.skipUnless(torch.cuda.is_available(), "needs CUDA") +class TestFusedRmsNormModulationGuard(unittest.TestCase): + """The guard answers exactly the question the launcher would raise on.""" + + def setUp(self): + torch.manual_seed(0) + self.x = torch.randn(2, 16, 2048, device="cuda", dtype=torch.bfloat16) + self.rows = torch.randn(2, 1, 2048, device="cuda", dtype=torch.bfloat16) + + def test_admits_the_layout_the_kernel_serves(self): + self.assertTrue( + can_use_fused_rmsnorm_modulation(self.x, self.rows, self.rows.clone()) + ) + + def test_rejects_modulation_rows_that_are_not_contiguous(self): + # LTX-2.5's shape: the rows arrive as a stride into a wider tensor. + wide = torch.randn(2, 1, 4096, device="cuda", dtype=torch.bfloat16) + strided = wide[:, :, ::2] + self.assertFalse(strided.is_contiguous()) + self.assertFalse(can_use_fused_rmsnorm_modulation(self.x, strided, self.rows)) + + def test_rejects_a_per_token_modulation(self): + per_token = torch.randn(2, 16, 2048, device="cuda", dtype=torch.bfloat16) + self.assertFalse(can_use_fused_rmsnorm_modulation(self.x, per_token, self.rows)) + + def test_rejects_mismatched_dtype_and_width(self): + self.assertFalse( + can_use_fused_rmsnorm_modulation(self.x, self.rows.float(), self.rows) + ) + narrow = torch.randn(2, 16, 1024, device="cuda", dtype=torch.bfloat16) + rows = torch.randn(2, 1, 1024, device="cuda", dtype=torch.bfloat16) + self.assertFalse(can_use_fused_rmsnorm_modulation(narrow, rows, rows)) + + def test_whatever_the_guard_rejects_the_launcher_would_have_raised(self): + # The guard is only worth having if it is exactly the launcher's + # contract: a rejected input must be one the launcher refuses, and an + # admitted one must go through. + from sglang.kernels.ops.diffusion import fused_rmsnorm_scale_shift_bitexact + + weight = torch.ones(2048, device="cuda", dtype=torch.bfloat16) + wide = torch.randn(2, 1, 4096, device="cuda", dtype=torch.bfloat16) + for name, scale, shift in ( + ("non-contiguous rows", wide[:, :, ::2], self.rows), + ("per-token modulation", self.x, self.rows), + ("fp32 rows", self.rows.float(), self.rows), + ): + with self.subTest(name): + self.assertFalse(can_use_fused_rmsnorm_modulation(self.x, scale, shift)) + with self.assertRaises(RuntimeError): + fused_rmsnorm_scale_shift_bitexact( + self.x, weight, scale, shift, 1e-6 + ) + + self.assertTrue(can_use_fused_rmsnorm_modulation(self.x, self.rows, self.rows)) + fused_rmsnorm_scale_shift_bitexact(self.x, weight, self.rows, self.rows, 1e-6) + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-v"])) diff --git a/python/sglang/multimodal_gen/test/unit/test_minimax_h3_admission.py b/python/sglang/multimodal_gen/test/unit/test_minimax_h3_admission.py index a3cfe95c26..157c98c46b 100644 --- a/python/sglang/multimodal_gen/test/unit/test_minimax_h3_admission.py +++ b/python/sglang/multimodal_gen/test/unit/test_minimax_h3_admission.py @@ -248,7 +248,7 @@ def test_hybrid_override_loads_ref_config_and_admits_all_native_tasks(weights): stage = MiniMaxH3PartitionAdmissionStage(pipeline.release_metadata) for task in ("t2va", "fl2va", "ref2va"): batch = SimpleNamespace( - sampling_params=SimpleNamespace(task=task, quality="lossless"), + sampling_params=SimpleNamespace(task=task, quality="exact"), num_inference_steps=50, ) assert ( @@ -440,7 +440,7 @@ def test_admission_rejects_steps_exceeding_online_adaln_gpu_plans(): server_args = _quality_server_args() server_args.minimax_h3_adaln_online = True batch = SimpleNamespace( - sampling_params=SimpleNamespace(task="t2va", quality="lossless"), + sampling_params=SimpleNamespace(task="t2va", quality="exact"), num_inference_steps=50, is_warmup=False, ) @@ -454,15 +454,15 @@ def test_admission_rejects_steps_exceeding_online_adaln_gpu_plans(): assert stage.forward(batch, server_args) is batch -def test_extra_high_quality_does_not_enable_h3_cache_dit(): +def test_lossless_quality_does_not_enable_h3_cache_dit(): stage = MiniMaxH3DenoisingStage.__new__(MiniMaxH3DenoisingStage) stage.server_args = SimpleNamespace(enable_breakable_cuda_graph=False) stage._cache_dit_enabled = False stage._minimax_h3_cache_mode = None - stage._minimax_h3_quality = "lossless" + stage._minimax_h3_quality = "exact" batch = SimpleNamespace( sampling_params=SimpleNamespace( - quality="extra-high", + quality="lossless", _explicit_fields={"quality"}, enable_cache_dit=None, cache_dit_params=None, @@ -475,7 +475,7 @@ def test_extra_high_quality_does_not_enable_h3_cache_dit(): with patch.object(DenoisingStage, "_cache_dit_requested", return_value=True): stage._maybe_enable_cache_dit(50, batch) - assert stage._minimax_h3_quality == "extra-high" + assert stage._minimax_h3_quality == "lossless" assert stage._minimax_h3_cache_mode is None assert not stage._cache_dit_enabled @@ -534,7 +534,7 @@ def test_quality_admission_fails_closed_outside_validated_request(): server_args.attention_backend = "sage_attn" assert stage.forward(batch, server_args) is batch - batch.sampling_params.quality = "extra-high" + batch.sampling_params.quality = "lossless" assert stage.forward(batch, server_args) is batch batch.sampling_params.quality = "ultra" diff --git a/python/sglang/multimodal_gen/test/unit/test_numerics_policy.py b/python/sglang/multimodal_gen/test/unit/test_numerics_policy.py new file mode 100644 index 0000000000..39b9aa1d0a --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_numerics_policy.py @@ -0,0 +1,60 @@ +# SPDX-License-Identifier: Apache-2.0 +"""The worker states its approximate-numerics settings instead of inheriting them. + +TF32 convolutions and bf16 reduced-precision reduction change what is computed, +so leaving them at whatever the installed PyTorch defaults to makes the exact +tier's contract depend on the build. The policy sets both explicitly. +""" + +import sys +import unittest + +import pytest +import torch + +from sglang.multimodal_gen.runtime.utils.numerics_policy import apply_numerics_policy + + +class TestNumericsPolicy(unittest.TestCase): + def setUp(self): + self._cudnn_tf32 = torch.backends.cudnn.allow_tf32 + self._bf16_reduction = ( + torch.backends.cuda.matmul.allow_bf16_reduced_precision_reduction + ) + + def tearDown(self): + torch.backends.cudnn.allow_tf32 = self._cudnn_tf32 + torch.backends.cuda.matmul.allow_bf16_reduced_precision_reduction = ( + self._bf16_reduction + ) + + def test_both_switches_follow_the_policy(self): + for tf32, bf16 in ((False, False), (True, True), (False, True)): + with self.subTest(tf32=tf32, bf16=bf16): + apply_numerics_policy( + allow_cudnn_tf32=tf32, + allow_bf16_reduced_precision_reduction=bf16, + ) + self.assertEqual(torch.backends.cudnn.allow_tf32, tf32) + self.assertEqual( + torch.backends.cuda.matmul.allow_bf16_reduced_precision_reduction, + bf16, + ) + + def test_the_strict_setting_is_reachable_from_any_starting_state(self): + torch.backends.cudnn.allow_tf32 = True + torch.backends.cuda.matmul.allow_bf16_reduced_precision_reduction = True + + apply_numerics_policy( + allow_cudnn_tf32=False, + allow_bf16_reduced_precision_reduction=False, + ) + + self.assertFalse(torch.backends.cudnn.allow_tf32) + self.assertFalse( + torch.backends.cuda.matmul.allow_bf16_reduced_precision_reduction + ) + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-v"])) diff --git a/python/sglang/multimodal_gen/test/unit/test_openai_image_api.py b/python/sglang/multimodal_gen/test/unit/test_openai_image_api.py index 492d065772..1f6c40ecb9 100644 --- a/python/sglang/multimodal_gen/test/unit/test_openai_image_api.py +++ b/python/sglang/multimodal_gen/test/unit/test_openai_image_api.py @@ -64,7 +64,7 @@ def test_runtime_sampling_quality_preserves_the_openai_default(): assert _runtime_sampling_quality(None) is None assert _runtime_sampling_quality("auto") is None assert _runtime_sampling_quality("lossless") == "lossless" - assert _runtime_sampling_quality("extra-high") == "extra-high" + assert _runtime_sampling_quality("lossless") == "lossless" assert _runtime_sampling_quality("high") == "high" diff --git a/python/sglang/multimodal_gen/test/unit/test_precision_consistency.py b/python/sglang/multimodal_gen/test/unit/test_precision_consistency.py index 732bb5d570..1b35465e71 100644 --- a/python/sglang/multimodal_gen/test/unit/test_precision_consistency.py +++ b/python/sglang/multimodal_gen/test/unit/test_precision_consistency.py @@ -94,14 +94,14 @@ class TestDiffusionPrecisionConsistency(unittest.TestCase): self.assertEqual( resolve_decode_precision( self._server_args(vae_decode_precision_high="bf16"), - quality="extra-high", + quality="lossless", ), torch.float16, ) self.assertEqual( resolve_decode_precision( self._server_args(vae_decode_precision_high="bf16"), - quality="lossless", + quality="exact", ), torch.float16, ) diff --git a/python/sglang/multimodal_gen/test/unit/test_quality_tier_admission.py b/python/sglang/multimodal_gen/test/unit/test_quality_tier_admission.py new file mode 100644 index 0000000000..1ec3ba4a67 --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_quality_tier_admission.py @@ -0,0 +1,168 @@ +# SPDX-License-Identifier: Apache-2.0 +"""The lossless-tier admission template admits re-association, not precision loss. + +Each fusion used to carry its own hand-picked atol/rtol, which cannot tell a +reassociated reduction from a kernel that quietly dropped a few mantissa bits: +both land "close". The template scores every implementation against the same +fp64 evaluation and judges the candidate relative to the path the model +already runs, so the question it answers is the tier's own question. +""" + +import sys +import unittest + +import pytest +import torch + +from sglang.multimodal_gen.test.quality_tier_admission import ( + assert_error_no_worse_than_reference, + assert_special_values_match, + discrete_flip_rate, + score_against, +) + + +def _row_sums(x: torch.Tensor, dtype: torch.dtype) -> torch.Tensor: + """Sequential row sum in ``dtype`` -- the association under test.""" + return x.to(dtype).sum(dim=-1) + + +class TestOperatorLevelGate(unittest.TestCase): + def setUp(self): + torch.manual_seed(0) + # Outlier channels are the point: DiT activations have them, and they + # decide whether a reassociated reduction stays within the reference. + self.x = torch.randn(64, 512, dtype=torch.float32) + self.x[:, ::97] *= 300.0 + self.reference_fp64 = self.x.to(torch.float64).sum(dim=-1) + + def test_admits_a_reassociated_reduction(self): + baseline = _row_sums(self.x, torch.float32) + # Pairwise summation: same operands and accumulation precision, a + # different order -- the shape of a lossless-tier fusion. + pairwise = self.x.to(torch.float32).reshape(64, -1, 2).sum(dim=-1).sum(dim=-1) + + base, cand = assert_error_no_worse_than_reference( + reference_fp64=self.reference_fp64, + baseline=baseline, + candidate=pairwise, + ) + self.assertGreater(base.rms, 0.0) + self.assertLess(cand.rms, base.rms * 1.5) + + def test_rejects_a_reduction_that_accumulates_in_bf16(self): + baseline = _row_sums(self.x, torch.float32) + degraded = _row_sums(self.x, torch.bfloat16).to(torch.float32) + + with self.assertRaisesRegex(AssertionError, "not lossless-tier"): + assert_error_no_worse_than_reference( + reference_fp64=self.reference_fp64, + baseline=baseline, + candidate=degraded, + ) + + def test_rejects_a_rounding_that_always_leans_one_way(self): + # A one-sided error small enough to stay inside the magnitude budget: + # the shape of truncation instead of round-to-nearest-even, or of an + # approximate transcendental whose error keeps its sign. It cannot be + # caught by a tolerance, because per-element it is tiny -- but it + # accumulates over layers and steps where a symmetric error cancels. + baseline = _row_sums(self.x, torch.float32) + leaning = baseline + 5e-5 + + base = score_against(self.reference_fp64, baseline) + cand = score_against(self.reference_fp64, leaning) + self.assertLess(cand.rms, base.rms * 1.5, "stays inside the magnitude budget") + + with self.assertRaisesRegex(AssertionError, "biased rounding"): + assert_error_no_worse_than_reference( + reference_fp64=self.reference_fp64, + baseline=baseline, + candidate=leaning, + ) + + def test_score_reports_signed_mean_separately_from_magnitude(self): + score = score_against( + torch.zeros(4, dtype=torch.float64), torch.full((4,), 0.5) + ) + self.assertAlmostEqual(score.mean, 0.5) + self.assertAlmostEqual(score.rms, 0.5) + self.assertAlmostEqual(score.max_abs, 0.5) + + +class TestSpecialValues(unittest.TestCase): + def test_a_fast_path_that_swallows_nan_is_rejected(self): + cases = {"nan row": (torch.tensor([1.0, float("nan"), 3.0]),)} + + assert_special_values_match( + baseline=torch.relu, candidate=torch.relu, cases=cases + ) + + with self.assertRaisesRegex(AssertionError, "moves NaNs"): + assert_special_values_match( + baseline=torch.relu, + candidate=lambda x: torch.nan_to_num(torch.relu(x)), + cases=cases, + ) + + def test_a_fast_path_that_ignores_strides_is_rejected(self): + # Reading the wrong elements is a magnitude error, not a structural + # one, so it is the operator-level gate that has to catch it. + torch.manual_seed(0) + strided = torch.randn(8, 16)[:, ::2] + reference = (strided.double() * 2).contiguous() + baseline = strided * 2 + # Read the same storage as if it were contiguous -- what a kernel that + # takes a data pointer and assumes packed rows actually does. (A + # `.flatten()` would quietly copy and get the right answer.) + ignores_stride = ( + torch.as_strided(strided, strided.shape, (strided.shape[1], 1)) * 2 + ) + + with self.assertRaisesRegex(AssertionError, "not lossless-tier"): + assert_error_no_worse_than_reference( + reference_fp64=reference, + baseline=baseline, + candidate=ignores_stride, + label="stride-ignoring kernel", + ) + + def test_a_rounding_level_difference_is_not_a_special_value_failure(self): + # The gate asks whether the special values behave the same, not + # whether the numbers match: a lossless-tier path differs there by + # construction, and judging that is the operator gate's job. + torch.manual_seed(0) + x = torch.randn(32, 8) + cases = {"plain": (x,), "with a NaN": (torch.cat([x[:1] * float("nan"), x]),)} + + assert_special_values_match( + baseline=lambda t: t * 3.0, + candidate=lambda t: (t.to(torch.bfloat16) * 3.0).to(t.dtype), + cases=cases, + ) + + def test_raising_where_the_reference_returns_is_rejected(self): + cases = {"empty": (torch.zeros(0),)} + + def explodes(x): + raise RuntimeError("no empty support") + + with self.assertRaisesRegex(AssertionError, "RuntimeError"): + assert_special_values_match( + baseline=lambda x: x, candidate=explodes, cases=cases + ) + + +class TestDiscreteDownstream(unittest.TestCase): + def test_flip_rate_counts_decisions_not_distance(self): + baseline = torch.tensor([0, 1, 2, 3]) + self.assertEqual(discrete_flip_rate(baseline, baseline.clone()), 0.0) + self.assertEqual(discrete_flip_rate(baseline, torch.tensor([0, 1, 2, 9])), 0.25) + + def test_empty_input_has_no_flips(self): + empty = torch.zeros(0, dtype=torch.long) + self.assertEqual(discrete_flip_rate(empty, empty), 0.0) + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-v"])) diff --git a/python/sglang/multimodal_gen/test/unit/test_quality_tier_boundary.py b/python/sglang/multimodal_gen/test/unit/test_quality_tier_boundary.py new file mode 100644 index 0000000000..175465513e --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_quality_tier_boundary.py @@ -0,0 +1,138 @@ +# SPDX-License-Identifier: Apache-2.0 +"""The tier boundary holds for the shapes the request-gated fusions actually use. + +`_QUALITY_FUSION_HANDLERS` assigns each fusion a tier from what it does to the +numbers. This pins that reading to measurement: the transformations the +lossless-tier fusions perform pass the admission gate, and the ones this +repository classifies as approximate fail it, on activations shaped like a +DiT's -- outlier channels included, because they are what decides whether a +reassociated reduction stays within the reference's error. + +Measured on real FLUX.1-dev MLP activations (4096x3072 bf16, max/p99 = 8.1) +the two groups separate by three to six orders of magnitude, so these verdicts +do not sit near the 1.5x threshold. +""" + +import sys +import unittest + +import pytest +import torch + +from sglang.multimodal_gen.test.quality_tier_admission import ( + assert_error_no_worse_than_reference, +) + + +def _dit_activations(rows: int, cols: int) -> torch.Tensor: + """BF16 activations with the outlier channels a DiT block carries.""" + generator = torch.Generator(device="cuda").manual_seed(0) + x = torch.randn(rows, cols, device="cuda", dtype=torch.float32, generator=generator) + x[:, ::97] *= 8.0 + return x.to(torch.bfloat16) + + +@unittest.skipUnless(torch.cuda.is_available(), "needs CUDA") +class TestLosslessTierTransformations(unittest.TestCase): + """What the lossless-tier fusions do stays inside the gate.""" + + def test_a_gemm_epilogue_fusion_is_admitted(self): + # fused linear+GELU: the cublasLt epilogue rounds once where the + # reference rounds the GEMM output and then the GELU output. + x = _dit_activations(1024, 1024) + weight = _dit_activations(2048, 1024) + bias = _dit_activations(1, 2048)[0] + + reference = torch.nn.functional.gelu( + x.double() @ weight.double().t() + bias.double(), approximate="tanh" + ) + baseline = torch.nn.functional.gelu( + torch.nn.functional.linear(x, weight, bias), approximate="tanh" + ) + candidate = torch._addmm_activation(bias, x, weight.t(), use_gelu=True) + + assert_error_no_worse_than_reference( + reference_fp64=reference, + baseline=baseline, + candidate=candidate, + label="fused linear+GELU", + ) + + def test_a_reassociated_reduction_is_admitted(self): + x = _dit_activations(512, 2048).float() + + assert_error_no_worse_than_reference( + reference_fp64=x.double().sum(dim=-1), + baseline=x.sum(dim=-1), + candidate=x.reshape(x.shape[0], -1, 2).sum(-1).sum(-1), + label="reassociated fp32 reduction", + ) + + def test_the_gated_residual_rounding_is_admitted(self): + # Helios / LingBot: the reference multiplies the FP32 gate and update + # and rounds once at the end; the fused kernel needs one dtype, so it + # rounds both to BF16 first. One rounding moves earlier -- which the + # taxonomy can only call equivalent if it measures that way. + residual = _dit_activations(1024, 1024) + update = _dit_activations(1024, 1024).float() + gate = torch.rand(1024, 1, device="cuda") * 0.2 + + assert_error_no_worse_than_reference( + reference_fp64=residual.double() + update.double() * gate.double(), + baseline=residual + (update * gate).to(residual.dtype), + candidate=residual + (update.to(residual.dtype) * gate.to(residual.dtype)), + label="per-token gated residual", + ) + + +@unittest.skipUnless(torch.cuda.is_available(), "needs CUDA") +class TestApproximateTransformationsAreRejected(unittest.TestCase): + """What the high-tier fusions do cannot pass as a rounding difference.""" + + def _assert_rejected(self, label, reference_fp64, baseline, candidate): + with self.assertRaisesRegex(AssertionError, "not lossless-tier"): + assert_error_no_worse_than_reference( + reference_fp64=reference_fp64, + baseline=baseline, + candidate=candidate, + label=label, + ) + + def test_norm_statistics_rounded_through_bf16(self): + # Ideogram's BF16-native gate RMSNorm against F.rms_norm's fp32 stats. + x = _dit_activations(512, 2048) + self._assert_rejected( + "bf16 norm statistics", + x.double().pow(2).mean(dim=-1), + x.float().pow(2).mean(dim=-1), + x.pow(2).mean(dim=-1).float(), + ) + + def test_a_gemm_fed_bf16_where_the_reference_promotes_to_fp32(self): + # SANA-Video's BF16-input linear attention. + a = _dit_activations(512, 1024) + b = _dit_activations(512, 1024).t().contiguous() + self._assert_rejected( + "BF16-input attention GEMM", + a.double() @ b.double(), + a.float() @ b.float(), + (a @ b).float(), + ) + + def test_quantizing_before_the_reference_intermediate_exists(self): + # FLUX.2's NVFP4 FC1+SwiGLU+quant. + x = _dit_activations(512, 2048) + scale = x.abs().amax(dim=-1, keepdim=True).float() / 448.0 + quantized = (x.float() / scale).clamp(-448, 448).to( + torch.float8_e4m3fn + ).float() * scale + self._assert_rejected( + "quantized FC2 input", + torch.nn.functional.silu(x.double()), + torch.nn.functional.silu(x.float()), + torch.nn.functional.silu(quantized), + ) + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-v"])) diff --git a/python/sglang/multimodal_gen/test/unit/test_sampling_params.py b/python/sglang/multimodal_gen/test/unit/test_sampling_params.py index d708908ea1..e33c739110 100644 --- a/python/sglang/multimodal_gen/test/unit/test_sampling_params.py +++ b/python/sglang/multimodal_gen/test/unit/test_sampling_params.py @@ -30,7 +30,8 @@ from sglang.multimodal_gen.configs.sample.sampling_params import ( SamplingParams, SkipSoftmaxParams, _json_safe, - quality_allows_kernel_fusions, + normalize_quality, + quality_allows, resolve_skip_softmax_params, ) from sglang.multimodal_gen.configs.sample.spectrum import SpectrumParams @@ -54,17 +55,30 @@ class TestSamplingParamsValidate(unittest.TestCase): SamplingParams(num_outputs_per_prompt=0) def test_quality_defaults_to_lossless(self): + # The default runs every fast path that keeps the reference math; + # "exact" is the opt-in for bit-identical output. self.assertEqual(SamplingParams().quality, "lossless") def test_quality_levels_are_cumulative(self): - self.assertEqual(QUALITY_LEVELS, ("lossless", "extra-high", "high")) + self.assertEqual(QUALITY_LEVELS, ("exact", "lossless", "high")) for quality in QUALITY_LEVELS: with self.subTest(quality=quality): self.assertEqual(SamplingParams(quality=quality).quality, quality) - self.assertFalse(quality_allows_kernel_fusions("lossless")) - self.assertTrue(quality_allows_kernel_fusions("extra-high")) - self.assertTrue(quality_allows_kernel_fusions("high")) + # A request admits the fast paths of its own tier and of every + # stricter one, so the levels stay cumulative. + for request, admits in ( + ("exact", {"exact"}), + ("lossless", {"exact", "lossless"}), + ("high", {"exact", "lossless", "high"}), + ): + for tier in QUALITY_LEVELS: + with self.subTest(request=request, tier=tier): + self.assertEqual(quality_allows(request, tier), tier in admits) + + def test_extra_high_is_accepted_as_the_former_name_of_lossless(self): + self.assertEqual(SamplingParams(quality="extra-high").quality, "lossless") + self.assertEqual(normalize_quality("extra-high"), "lossless") def test_quality_rejects_invalid_values(self): for bad in ("ultra", "draft", "fast", "", True, 1): @@ -371,7 +385,7 @@ class TestSamplingParamsCliArgs(unittest.TestCase): def test_quality_is_request_scoped_cli_arg(self): self.assertNotIn("quality", self._parse_cli_kwargs([])) - for quality in ("extra-high", "high"): + for quality in ("exact", "lossless", "high", "extra-high"): with self.subTest(quality=quality): self.assertEqual( self._parse_cli_kwargs(["--quality", quality])["quality"], quality