[diffusion] feat: name the quality tiers after what they guarantee (#42370)

Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Mick
2026-10-07 09:49:30 +08:00
committed by GitHub
co-authored by Mick Qian Claude Opus 5
parent 8f1790f595
commit 7a4d6dfd2f
65 changed files with 1272 additions and 270 deletions
+12 -7
View File
@@ -60,11 +60,12 @@ themselves against the live eager chain on first sight via
`sites/bitexact_gate.py` and fall back permanently on mismatch — the
dispatch they replicate can change under them.
**Not bit-exact → quality-gated.** Mounted onto marked `nn.Module` sites only
for `quality="extra-high"` and `quality="high"` requests, at batch boundaries,
all-or-nothing per transformer (`sites/quality_gate.py`). `extra-high` adds
only these request-gated DiT/VAE fusions; `high` is cumulative and may also
enable model-owned approximate paths such as Cache-DiT or a lower-precision
**Not bit-exact → quality-gated.** Mounted onto marked `nn.Module` sites at
batch boundaries, all-or-nothing per transformer (`sites/quality_gate.py`),
from the lowest tier that may run each one: `lossless` for a fusion that keeps
the reference math and every operand's precision and only moves the rounding,
`high` for one that quantizes or lowers a precision. `high` is cumulative and
may also enable model-owned approximate paths such as Cache-DiT or a lower-precision
decode. A plain fp32 single-pass norm fusion looks harmless and is not: on
ERNIE-Image it moved the 50-step trajectory to PSNR 18.83 dB, which is what
motivated the bit-exact rewrite.
@@ -248,7 +249,11 @@ inspecting model modules is its whole job.
wrapper when the fallback policy is shared.
4. State the numerical contract in the module docstring, including which
shapes it was verified on.
5. If it is not bit-exact, gate it through `sites/`. It must mount for both
`extra-high` and `high`, never for the default `lossless` path.
5. If it is not bit-exact, gate it through `sites/` and declare its tier in
`_QUALITY_FUSION_HANDLERS`: `lossless` when it keeps the reference math and
every operand's precision, `high` when it does not. Never `exact`, which
is reserved for paths that reproduce the reference's rounding. A
`lossless` claim has to pass
`multimodal_gen/test/quality_tier_admission.py`.
6. Test it in the domain suite (`test/registered/kernels/ops/diffusion/`), and
the model wiring in `test_model_fast_paths.py`.
@@ -642,6 +642,7 @@ _EXPORTS: dict[str, str] = {
"can_use_flux2_strided_qknorm_rope": "rope.flux2_qknorm_rope_triton",
"flux2_strided_qknorm_rope": "rope.flux2_qknorm_rope_triton",
"rmsnorm_preserve_reduction": "norm.rmsnorm_preserve_reduction",
"can_use_fused_rmsnorm_modulation": "norm.rmsnorm_scale_shift_bitexact",
"can_use_fused_rmsnorm_scale_shift": "norm.rmsnorm_scale_shift_bitexact",
"fused_rmsnorm_scale_shift_bitexact": "norm.rmsnorm_scale_shift_bitexact",
"fused_scale_residual_rmsnorm_scale_shift_bitexact": "norm.rmsnorm_scale_shift_bitexact",
@@ -164,6 +164,32 @@ def _is_row_broadcast(t: torch.Tensor, shape: tuple, device: torch.device) -> bo
)
def can_use_fused_rmsnorm_modulation(
x: torch.Tensor, scale: torch.Tensor, shift: torch.Tensor
) -> bool:
"""Whether this kernel can serve ``x`` with this ``scale``/``shift``.
The launcher raises on everything it cannot serve, which is the right
answer for a caller that has already committed. A request-gated caller has
not: a fast path that meets an input it cannot handle has to fall back,
not fail the request. So it asks here first, and this mirrors exactly what
:func:`_validate_rmsnorm_modulate` checks about ``x`` and the modulation
rows -- a dtype-and-width check alone leaves the layout of ``scale`` and
``shift`` to chance.
"""
if not (
can_use_fused_rmsnorm_scale_shift(x.dtype, x.shape[-1])
and x.is_cuda
and x.ndim == 3
and x.is_contiguous()
):
return False
row_shape = (x.shape[0], 1, x.shape[-1])
return _is_row_broadcast(scale, row_shape, x.device) and _is_row_broadcast(
shift, row_shape, x.device
)
def can_use_fused_rmsnorm_scale_shift(dtype: torch.dtype, hidden: int) -> bool:
"""Select the BF16 FlashInfer reduction replicated by this kernel."""
return (
@@ -15,7 +15,7 @@ eager LingBot chain ``(norm(x) * (1 + scale) + shift).to(dtype)``.
Unlike the bit-exact ``rmsnorm_scale_shift_bitexact`` (ERNIE), this kernel does
*not* reproduce PyTorch's parallel variance reduction order bit-for-bit, so it
is intended for the request-gated (``quality="extra-high"``/``"high"``) fusion
is intended for the request-gated (``quality="lossless"``/``"high"``) fusion
path, matching the existing quality-gated LingBot RMSNorm fusion.
"""
@@ -63,6 +63,13 @@ def flashinfer_rmsnorm_diagnostic_hint() -> str:
class BitExactFusionGate:
"""Track permanent disable + first-sight ``torch.equal`` verification.
This is the ``exact`` tier's mechanism: it backs a fast path that is
bit-exact by construction, so the path mounts unconditionally and the gate
only catches the case where the dispatch it replicates changed underneath
it. A fast path whose rounding differs from the reference belongs to a
later tier and uses ``QualityGatedFusion`` instead, which mounts per
request from the tier declared in ``_QUALITY_FUSION_HANDLERS``.
Two modes:
* **once-for-all** (default): the first successful equal-check enables the
@@ -2,7 +2,7 @@
The fused FC1 + SwiGLU + FC2-input quantization path changes the rounding
order by quantizing before the reference BF16 intermediate is materialized.
Keep it disabled for the lossless default and mount it only for
Keep it disabled below the approximate tier and mount it only for
``quality="high"`` requests at denoising batch boundaries.
"""
@@ -14,7 +14,7 @@ statistics round through bf16) but **not bit-exact**. Following the
fused-linear-GELU precedent, sites are therefore mounted only for
``quality="high"`` requests via :func:`mount_fused_gate_rmsnorm` /
:func:`unmount_fused_gate_rmsnorm` at batch boundaries; the default
``"lossless"`` path keeps the unmodified reference chain bit-for-bit.
``"exact"`` path keeps the unmodified reference chain bit-for-bit.
Mounting is all-or-nothing per transformer: if any marked site fails the
static guards (non-bf16 norm weight, hidden size above the kernel limit, ...)
@@ -7,8 +7,8 @@ The shared ``residual_gate_add`` kernel computes ``residual + update * gate``
in a single pass but requires one dtype, so the gate and update are first cast
to BF16. That reordering of the FP32 multiply is numerically equivalent only at
half-precision rounding level (not bit-exact), so the fusion is opt-in:
``quality="extra-high"`` and ``quality="high"`` mount it, while the default
``quality="lossless"`` keeps the reference FP32-multiply form bit-for-bit.
``quality="lossless"`` and ``quality="high"`` mount it, while the default
``quality="exact"`` keeps the reference FP32-multiply form bit-for-bit.
"""
from __future__ import annotations
@@ -7,8 +7,8 @@ the attention and FFN updates, where ``update`` (post-norm) and the per-token
pass but requires one dtype, so the gate and update are first cast to BF16.
That reordering of the FP32 multiply is numerically equivalent only at
half-precision rounding level (not bit-exact), so the fusion is opt-in:
``quality="extra-high"`` and ``quality="high"`` mount it, while the default
``quality="lossless"`` keeps the reference FP32-multiply form bit-for-bit.
``quality="lossless"`` and ``quality="high"`` mount it, while the default
``quality="exact"`` keeps the reference FP32-multiply form bit-for-bit.
"""
from __future__ import annotations
@@ -7,7 +7,7 @@ the reference formula still promotes their multiplication through the FP32
hidden states. For ``quality="high"``, existing diffusion Triton RMSNorm
kernels replace that chain. Wide rows with FP32 weights use the one-row
``norm_infer`` kernel; the remaining sites use the tiled one-pass kernel. Their
reduction order is not bit-exact, so the default ``quality="lossless"`` path
reduction order is not bit-exact, so the default ``quality="exact"`` path
remains unchanged.
"""
@@ -2,7 +2,7 @@
The fused JIT kernel is bit-exact with the local eager ``add + GELU`` chain for
eligible ModelOpt FP4 linears. It still changes the model's kernel schedule, so
keep the default ``quality="lossless"`` path unchanged and mount this fast path
keep the default ``quality="exact"`` path unchanged and mount this fast path
only for the existing ``quality="high"`` contract.
"""
@@ -2,7 +2,7 @@
Packing the three BF16 text projections into one GEMM changes the reduction
association and is therefore not bit-exact. The packed weights stay resident
for checkpoint compatibility, but ``quality="lossless"`` applies their three
for checkpoint compatibility, but ``quality="exact"`` applies their three
slices independently. ``quality="high"`` mounts the single-GEMM path.
"""
@@ -6,7 +6,7 @@ requesting an FP32 output/accumulator from cuBLAS. The second GEMM stays in
FP32. This removes two large dtype-conversion kernels and lets the first GEMM
use BF16 Tensor Cores, at the cost of half-precision input rounding.
The default ``quality="lossless"`` path remains the original FP32-input chain
The default ``quality="exact"`` path remains the original FP32-input chain
bit-for-bit. Only the single-batch CUDA layout used by native SANA-Video is
eligible; unsupported dtypes and layouts fall back to the reference path.
"""
@@ -657,7 +657,7 @@ BCG validation must prove all of the following:
For a non-bit-exact optimization, integrate through the request-scoped site
framework under `sglang.kernels.ops.diffusion.sites`. Mark sites during model
construction and let `QualityGatedFusion` mount them for both
`quality="extra-high"` and `quality="high"`; `quality="lossless"` must keep
`quality="lossless"` and `quality="high"`; `quality="lossless"` must keep
the original code path. A high-only sparse, caching, or other approximate
path must remain outside this fusion gate.
Eligibility must be all-or-nothing for coupled sites and fail closed on dtype,
@@ -666,7 +666,7 @@ guard/parity tests and a model wiring test instead of embedding request-policy
branches throughout the DiT.
Finally, use the benchmark/profile skill's `--quality-bcg-matrix` to run
same-GPU ABBA pairs for Eager/BCG at lossless/extra-high/high. Report denoise and saved
same-GPU ABBA pairs for Eager/BCG at exact/lossless/high. Report denoise and saved
request e2e separately, require at least 1.5% repeated mean e2e improvement for
an optimization PR, attach profile and generated-media A/B evidence, then
delete the task-owned checkpoint cache and verify zero residual weight files
@@ -9,7 +9,7 @@ Use this skill when measuring denoise performance, finding the slow op, checking
This skill is diagnosis-first. It owns:
- checked-in denoise benchmark presets
- same-GPU quality/BCG applicability checks with repeated lossless, extra-high, and high rows
- same-GPU quality/BCG applicability checks with repeated exact, lossless, and high rows
- perf dump collection and before/after comparison
- `torch.profiler` trace capture and quick hotspot ranking
- mapping hot kernels back to known fast paths and fusion families
@@ -65,15 +65,15 @@ Always rule out these existing families first:
- Z-Image bf16-native Triton RMSNorm scale/tanh-residual modulation
- SANA packed self-attention Q/K/V and cross-attention K/V GEMMs
- SANA-Video's packed projections and request-scoped BF16-input linear
attention at `quality=extra-high` or `quality=high`; keep the second attention GEMM in FP32 and
compare against `quality=lossless` before changing its precision further
attention at `quality=lossless` or `quality=high`; keep the second attention GEMM in FP32 and
compare against `quality=exact` before changing its precision further
- SANA-Video reuse of SANA's bit-exact bias/activation, residual-gate, and
LayerNorm-modulation fast paths before adding video-only kernels
- MiniMax-H3 indexed modulation, fused QK norm + RoPE, packed Ulysses QKV,
USP relayout, and batched TP AdaLN collectives
- bit-exact diffusion adaLN modulation and fused LayerNorm + modulation for
FLUX.1, GLM-Image, and SANA
- request-scoped DiT and VAE fast paths at `quality=extra-high` or `quality=high`
- request-scoped DiT and VAE fast paths at `quality=lossless` or `quality=high`
- LingBot Video's default-on fused group-limited top-k expert selection before
treating its router's `topk`/mask/gather chain as a new hotspot
- Wan causal-VAE cache/padding and DupUp3D data-movement fusions
@@ -96,12 +96,12 @@ controlled comparator, never for the eager ground truth. The legacy
`--no-torch-compile` spelling remains accepted but is redundant.
For kernel/BCG discovery, run `--quality-bcg-matrix`. It executes Eager/BCG as
A-B-B-A at `lossless`, then repeats the pair at `extra-high` and `high`, on
one locked GPU set and one isolated checkpoint cache. The extra-high/high+BCG
A-B-B-A at `exact`, then repeats the pair at `lossless` and `high`, on
one locked GPU set and one isolated checkpoint cache. The lossless/high+BCG
rows are applicability checks, not presumed-valid performance cells. A BCG row is invalid unless the log
contains `[Diffusion BCG] captured` and contains no support-disable,
capture-failure, serving-signature-miss, or late quality-fusion marker. In
particular, a request-scoped DiT fusion mounted after lossless warmup capture
particular, a request-scoped DiT fusion mounted after exact warmup capture
would be bypassed by replay; reject that row even when capture and signature
checks pass. For video presets, the helper declares both the request resolution
and `--warmup-num-frames` so the synthetic BCG warmup captures the requested
@@ -122,7 +122,7 @@ its normal cleanup finally block without modifying the seed cache.
Keep prompt, negative prompt, seed, shape, steps, guidance, dtype, topology,
and residency fixed. Lossless comparisons require byte-identical artifacts.
For `quality=extra-high` and `quality=high`, report aggregate and worst-frame SSIM/PSNR; the repository
For `quality=lossless` and `quality=high`, report aggregate and worst-frame SSIM/PSNR; the repository
defaults are 0.95/28 dB for images and 0.92/24 dB for video unless the model's
checked-in consistency metadata defines a different threshold. A performance
PR needs repeated saved-request e2e improvement of at least 1.5%, a
@@ -139,9 +139,9 @@ The helper defaults to eager. Add `--torch-compile` only for a labeled compile
control. `--no-torch-compile` remains accepted for compatibility but is no
longer required.
Run one explicit quality or BCG comparator with `--quality {lossless,extra-high,high}` and
Run one explicit quality or BCG comparator with `--quality {exact,lossless,high}` and
`--breakable-cuda-graph`. BCG and `torch.compile` are intentionally mutually
exclusive in this helper. An extra-high/high+BCG command is only a compatibility probe:
exclusive in this helper. An lossless/high+BCG command is only a compatibility probe:
it is invalid if request-scoped DiT fusions mount after the lossless warmup
graphs were captured. When a preset has explicit width and height, the helper
declares that same `--warmup-resolutions` value automatically. Video presets
@@ -150,16 +150,16 @@ with an explicit frame count also declare the matching `--warmup-num-frames`:
```bash
PYTHONPATH=python python3 "$BENCH_PY" \
--model longcat-image \
--quality extra-high \
--quality lossless \
--breakable-cuda-graph \
--label bcg-extra-high \
--label bcg-lossless \
--output-dir "${BENCH_DIR}"
```
For optimization discovery, use the full repeated matrix. It runs
Eager/BCG/BCG/Eager at `lossless`, then the same sequence at `extra-high` and
Eager/BCG/BCG/Eager at `exact`, then the same sequence at `lossless` and
`high`, while holding one GPU set and one isolated checkpoint cache. The
extra-high/high+BCG cells test
lossless/high+BCG cells test
whether the combination is actually supported; do not average them when the
runtime rejects the combination or the helper detects a late quality-fusion
mount. The helper hashes every generated image, video, audio, or 3D mesh
@@ -321,13 +321,13 @@ Use the preset categories this way:
| `qwen-edit-base` | `Qwen/Qwen-Image-Edit` | No | Covers the original native `QwenImageEditPipelineConfig`, which is distinct from the 2509/2511 edit-plus paths; public SGLang edit fixture, 1024x1024. |
| `qwen-image-layered` | `Qwen/Qwen-Image-Layered` | No | Native layered-image path using the same public reference image and four-frame request as the GPU server case, at the registered 640x640 canvas. |
| `stable-diffusion-3.5-medium` | `stabilityai/stable-diffusion-3.5-medium-diffusers` | No | Representative native `StableDiffusion3PipelineConfig` path at 1024x1024. The repository is gated, so export `HF_TOKEN`; an unauthenticated run is a recorded access blocker, not model evidence. |
| `sana-video` | `Efficient-Large-Model/SANA-Video_2B_480p_diffusers` | No | CI-sized T2V baseline: 832x480, 17 frames, 8 steps, guidance 6.0. The BCG comparator declares the same 17-frame warmup shape. Compare all three tiers; `extra-high` and `high` enable the BF16-input first linear-attention GEMM while retaining FP32 output and the FP32 second GEMM. |
| `sana-video` | `Efficient-Large-Model/SANA-Video_2B_480p_diffusers` | No | CI-sized T2V baseline: 832x480, 17 frames, 8 steps, guidance 6.0. The BCG comparator declares the same 17-frame warmup shape. Compare all three tiers; `lossless` and `high` enable the BF16-input first linear-attention GEMM while retaining FP32 output and the FP32 second GEMM. |
| `sana-wm-bidirectional` | `Efficient-Large-Model/SANA-WM_bidirectional` | No | Dense two-stage TI2V baseline at the native 1280x704 shape, 49 frames, 16 fps, 20 steps, guidance 4.5, and a 48-frame forward/left action program. Uses the shared cat fixture. |
| `sana-wm-streaming` | `Efficient-Large-Model/SANA-WM_streaming` | No | Matching offline chunk-causal two-stage baseline with the streaming DiT and chunked refiner enabled; uses the same shape, fixture, seed, and camera action for comparison. |
| `lingbot-video-moe` | `robbyant/lingbot-video-moe-30b-a3b` | No | One-GPU eager baseline using the CI structured-JSON caption, 384x640, 17 frames, 12 steps, and text-encoder CPU offload. |
| `lingbot-world` | `robbyant/lingbot-world-fast-diffusers` | No | One-H200 offline single-chunk profile for the registered causal DMD path: 832x480x9, four steps, guidance 1.0, the shared image fixture, and forward-camera actions for all nine frames. Keep stateful websocket latency as a separate metric. |
| `lingbot-world-v2` | `robbyant/lingbot-world-v2-14b-causal-fast-diffusers` | No | Matching controlled single-chunk profile for the separately registered v2 checkpoint. The fixed shape, action program, and schedule make v1/v2 hotspot comparisons reproducible without presenting one-chunk e2e as stateful realtime latency. |
| `fastwan21-t2v-1.3b` | `FastVideo/FastWan2.1-T2V-1.3B-Diffusers` | No | One-GPU 832x480, 61-frame, 3-step DMD baseline. The preset pins manual mode with a resident DiT so lossless/extra-high/high comparisons do not measure an offload-policy change. |
| `fastwan21-t2v-1.3b` | `FastVideo/FastWan2.1-T2V-1.3B-Diffusers` | No | One-GPU 832x480, 61-frame, 3-step DMD baseline. The preset pins manual mode with a resident DiT so exact/lossless/high comparisons do not measure an offload-policy change. |
| `wan21-t2v-1.3b` | `Wan-AI/Wan2.1-T2V-1.3B-Diffusers` | No | Registered one-GPU 832x480, 81-frame Wan2.1 baseline at 50 steps and guidance 3.0. Keep it separate from FastWan and TurboWan because the longer schedule changes the end-to-end weight of VAE optimizations. |
| `wan21-t2v-14b` | `Wan-AI/Wan2.1-T2V-14B-Diffusers` | No | Cookbook-aligned four-GPU CFG/Ulysses baseline at 832x480, 81 frames, 50 steps, and guidance 5.0. Text encoding stays CPU-offloaded as in the documented deployment command. |
| `wan21-i2v-14b-480p` | `Wan-AI/Wan2.1-I2V-14B-480P-Diffusers` | No | Four-GPU CFG/Ulysses image-conditioned baseline at 832x480, 81 frames, 50 steps, and guidance 5.0. Uses the shared cat fixture and its motion prompt. |
@@ -669,8 +669,8 @@ a zero-exit run if either the requested perf dump or generated media is absent:
some generation failures are reported through the response payload without a
nonzero process exit.
For `quality=lossless`, compare saved artifact hashes and require byte equality
for a claimed lossless fast path or BCG change. For `quality=extra-high` and
For `quality=exact`, compare saved artifact hashes and require byte equality
for a claimed lossless fast path or BCG change. For `quality=lossless` and
`quality=high`, keep the
lossless artifact as ground truth and report both aggregate and worst-frame
SSIM/PSNR. Repository defaults are SSIM 0.95 / PSNR 28 dB for images and SSIM
@@ -791,7 +791,7 @@ the known mainline families.
| `to_q -> to_k -> to_v` on NVFP4 or Nunchaku FLUX-family checkpoints | Treat as a packed-QKV fast-path miss or checkpoint-format mismatch |
| `rmsnorm_scale` or `rmsnorm_tanh_residual` missing on Z-Image | Check the bf16-native Triton eligibility guards before proposing a new fusion |
| FLUX.1, GLM-Image, or SANA shows separate LayerNorm plus adaLN elementwise kernels | Check the bit-exact `modulate_scale_shift` and `fused_layernorm_modulate` guards/self-test before proposing another norm fusion |
| `quality=extra-high` or `quality=high` shows the same FLUX/GLM DiT or FLUX-family/Wan VAE chain as `lossless` | Check whether the request-scoped quality gate mounted and whether every site passed its all-or-nothing compatibility checks |
| `quality=lossless` or `quality=high` shows the same FLUX/GLM DiT or FLUX-family/Wan VAE chain as `exact` | Check whether the request-scoped quality gate mounted and whether every site passed its all-or-nothing compatibility checks |
| LTX-2 split RoPE appears as a long PyTorch elementwise chain | Check the `apply_ltx2_split_rotary_emb` Triton path and its shape guards |
| Wan decode is dominated by causal `cat + pad + contiguous`, feature-cache copies, or `repeat_interleave + permute + add` | Check the bit-exact Wan causal-cache and DupUp3D data-movement kernels before writing a new decoder kernel |
| masked attention spends time packing/unpacking Q/K/V | Check whether fused varlen USP pack/scatter should have engaged |
@@ -829,7 +829,7 @@ This skill intentionally stops here. It tells you whether you are looking at:
- [ ] `compare_perf.py` table generated
- [ ] one representative `torch.profiler` trace saved
- [ ] hotspot classified against `existing-fast-paths.md`
- [ ] lossless artifact hash is exact; extra-high/high aggregate and worst-frame SSIM/PSNR pass the checked-in threshold
- [ ] lossless artifact hash is exact; lossless/high aggregate and worst-frame SSIM/PSNR pass the checked-in threshold
- [ ] reference image or start/middle/end video contact sheet checked visually
- [ ] any PR claim has repeated saved-request e2e improvement >= 1.5%
- [ ] task-owned checkpoint cache cleaned and ledger shows zero residual weight files
@@ -110,12 +110,12 @@ framework-specific optimization workflow.
check dtype, alignment, shape, BCG/compile context, and the one-time equality
self-test before proposing another fusion.
4. Request-scoped fusion gates at `quality=extra-high` or `quality=high`
4. Request-scoped fusion gates at `quality=lossless` or `quality=high`
- Locations: `quality_gate.py`, `fused_ln_modulate.py`, `denoising.py`,
`decoding.py`, `fast_path_gate.py`, `flux2_vae_cuda_opt.py`, and
`wan_vae_cuda_opt.py`.
- Behavior: `quality="lossless"` is the default exact reference path.
`quality="extra-high"` and `quality="high"` mount the same validated but
`quality="lossless"` and `quality="high"` mount the same validated but
non-bit-exact DiT fusions and decode-scoped VAE rewrites. `high` is
cumulative and may additionally enable model-owned approximate paths.
Mounting is all-or-nothing per
@@ -397,13 +397,13 @@ framework-specific optimization workflow.
**Request-Scoped DiT Fusions with Breakable CUDA Graphs**
- DiT sites at `quality=extra-high` or `quality=high` are mounted at a request boundary. BCG warmup uses
- DiT sites at `quality=lossless` or `quality=high` are mounted at a request boundary. BCG warmup uses
the model's lossless sampling default unless a quality-aware graph variant
was captured explicitly.
- A graph captured before the request-quality mount retains the lossless module
branches. Replaying it after the mount silently bypasses the requested fused
kernels even when the tensor signature matches.
- Workflow rule: an extra-high/high+BCG cell is valid only when the model has no
- Workflow rule: an lossless/high+BCG cell is valid only when the model has no
request-scoped DiT quality sites, or when logs prove those sites were mounted
before the matching graph capture. A mount after `[Diffusion BCG] captured`
invalidates the row; do not use its latency or output as request-quality
@@ -434,7 +434,7 @@ framework-specific optimization workflow.
selector when its CUDA layout guard passes. Check that guard before treating
the `topk`/mask chain as a new opportunity. Its fused Triton RMSNorm row
kernels remain request-gated by weight dtype and hidden size at
`quality=extra-high` and `quality=high`; check those separate quality-site
`quality=lossless` and `quality=high`; check those separate quality-site
guards before treating an expanded `pow/mean/rsqrt` chain as new work.
- LTX-2.5 reuses the mature LTX-2 DiT paths. Treat the optional diffusion
decoder separately: confirm NATTEN `na3d` is active, then inspect its
@@ -448,7 +448,7 @@ framework-specific optimization workflow.
- AdaLN modulation: `LayerNormScaleShift`, `RMSNormScaleShift`, `ScaleResidual*` in `layernorm.py`.
- Bit-exact adaLN modulation / LayerNorm folding: `modulate_scale_shift` and
`fused_layernorm_modulate` through `flux.py`, `glm_image.py`, and `sana.py`.
- Request-scoped extra-high/high acceleration: `QualityGatedFusion` in
- Request-scoped lossless/high acceleration: `QualityGatedFusion` in
`quality_gate.py`, `_maybe_toggle_quality_fusions` in `denoising.py`, and
`use_vae_fast_path` in `decoding.py`.
- Bit-exact first-sight verify/disable: `BitExactFusionGate` in
@@ -466,7 +466,7 @@ framework-specific optimization workflow.
- LingBot Video MoE routing: `LingBotVideoRouter._group_limited_topk` uses
`group_limited_topk` through the diffusion facade when the CUDA layout guard
passes, otherwise it retains the eager reference chain.
- LTX2 RMSNorm+modulate and FFN GELU epilogue under `quality="extra-high"` and `quality="high"`:
- LTX2 RMSNorm+modulate and FFN GELU epilogue under `quality="lossless"` and `quality="high"`:
`mark_ltx2_rms_norm_modulate_site` / `fused_ltx2_rms_norm_modulate` in
`kernels/ops/diffusion/sites/ltx2_rmsnorm_modulate_site.py` (mount-based
`QualityGatedFusion`, not a first-sight `BitExactFusionGate` — the fused
@@ -16,7 +16,7 @@ Usage:
# Opt in to a compile control (presets are eager by default)
python3 python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-benchmark-profile/scripts/bench_diffusion_denoise.py --model flux --torch-compile
# Check Eager/BCG at every request quality on one GPU set; extra-high/high
# Check Eager/BCG at every request quality on one GPU set; lossless/high
# + BCG are invalid when request-scoped DiT fusions mount only after the
# lossless graph capture.
python3 python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-benchmark-profile/scripts/bench_diffusion_denoise.py --model sana-video --quality-bcg-matrix --model-cache-root /task/model-caches --cleanup-model-cache
@@ -81,7 +81,7 @@ DIFFUSERS_FALLBACK_SIGNALS = (
"using diffusers backend",
"loaded diffusers pipeline",
)
BENCHMARK_QUALITY_LEVELS = ("lossless", "extra-high", "high")
BENCHMARK_QUALITY_LEVELS = ("exact", "lossless", "high")
BCG_CAPTURE_SIGNAL = "[diffusion bcg] captured"
BCG_INVALID_SIGNALS = (
"[diffusion bcg] capture failed",
@@ -92,14 +92,14 @@ BCG_INVALID_SIGNALS = (
)
BCG_LATE_QUALITY_FUSION_SIGNAL = "quality fusion mounted after BCG capture"
QUALITY_BCG_ABBA_MATRIX = (
("eager-exact-a", "exact", False),
("bcg-exact-a", "exact", True),
("bcg-exact-b", "exact", True),
("eager-exact-b", "exact", False),
("eager-lossless-a", "lossless", False),
("bcg-lossless-a", "lossless", True),
("bcg-lossless-b", "lossless", True),
("eager-lossless-b", "lossless", False),
("eager-extra-high-a", "extra-high", False),
("bcg-extra-high-a", "extra-high", True),
("bcg-extra-high-b", "extra-high", True),
("eager-extra-high-b", "extra-high", False),
("eager-high-a", "high", False),
("bcg-high-a", "high", True),
("bcg-high-b", "high", True),
@@ -1897,7 +1897,7 @@ def _run_benchmark_once_impl(
if BCG_CAPTURE_SIGNAL in lower_line:
bcg_capture_detected = True
if (
quality in {"extra-high", "high"}
quality in {"lossless", "high"}
and breakable_cuda_graph
and bcg_capture_detected
and "mounted " in lower_line
@@ -2315,7 +2315,7 @@ def main():
"--quality-bcg-matrix",
action="store_true",
help=(
"Run lossless/extra-high/high Eager-vs-BCG as three ABBA pairs "
"Run exact/lossless/high Eager-vs-BCG as three ABBA pairs "
"on one GPU set "
"and one task-owned model cache."
),
@@ -163,7 +163,7 @@ These options **trade output quality** for speed or VRAM savings. Results will d
| Option | CLI Flag / Env Var | What It Does | Speedup | Quality Impact / Limitations |
|---|---|---|---|---|
| **Request Quality Fast Paths** | `--quality {extra-high,high}` (`lossless` is default) | `extra-high` mounts only request-gated DiT/VAE fusions. `high` includes that complete set and may add model-owned approximate paths such as Cache-DiT or lower-precision decode. | Model- and shape-specific | Support is per model and may be a no-op. Keep `--quality lossless` as the A/B ground truth, then compare `extra-high` before `high` to isolate fusion wins. Report aggregate and worst-frame SSIM/PSNR for every non-bit-exact path; defaults are 0.95/28 dB for images and 0.92/24 dB for video unless checked-in model metadata overrides them. Do not confuse this with `--output-quality`, which controls file compression. |
| **Request Quality Fast Paths** | `--quality {lossless,high}` (`exact` is default) | `lossless` mounts only request-gated DiT/VAE fusions. `high` includes that complete set and may add model-owned approximate paths such as Cache-DiT or lower-precision decode. | Model- and shape-specific | Support is per model and may be a no-op. Keep `--quality lossless` as the A/B ground truth, then compare `lossless` before `high` to isolate fusion wins. Report aggregate and worst-frame SSIM/PSNR for every non-bit-exact path; defaults are 0.95/28 dB for images and 0.92/24 dB for video unless checked-in model metadata overrides them. Do not confuse this with `--output-quality`, which controls file compression. |
| **Approximate Attention** | Server-wide: `--attention-backend sage_attn` / `sage_attn_3` / `sliding_tile_attn` / `video_sparse_attn` / `sparse_video_gen_2_attn` / `vmoba_attn` / `sla_attn` / `sage_sla_attn`. Per-request (dense drop-ins only): `--attention-backend-override sage_attn` sampling param / API `extra_body` — valid values `fa`, `torch_sdpa`, `sage_attn`, `sage_attn_3`; rejected (with a log) under BCG, torch.compile, sparse server backends, or a non-ring-capable target with ring parallelism. | Replaces exact attention with approximate or sparse variants. `sage_attn`: INT8/FP8 quantized Q·K; `sliding_tile_attn`: spatial-temporal tile skipping; others: model-specific sparse patterns. | ~1.5–2x on attention (varies by backend) | Quality degradation varies by backend and model. `sage_attn` is the most general; sparse backends (`sliding_tile_attn`, `video_sparse_attn`, etc.) are video-model-specific, may require config files (e.g. `--mask-strategy-file-path` for STA), and are server-level only. Requires corresponding packages installed. |
| **Cache-DiT** | Native: per-request `--enable-cache-dit true\|false` + `--cache-dit-params <json>` (sampling params; also via API `extra_body`). `SGLANG_CACHE_DIT_ENABLED` / `SGLANG_CACHE_DIT_*` env vars are the server-wide defaults for requests that leave them unset. Diffusers backend: `--backend diffusers --cache-dit-config <yaml-or-json>` | Caches intermediate residuals across denoising steps and skips redundant computations via DBCache, TaylorSeer, and optional SCM. | ~1.5-2x on supported models | Quality depends on cache policy. Compatible with `--dit-layerwise-offload`: skipped blocks are not streamed, and the first layer after a skip may sync-load. Models that touch every layer before the block loop (for example a full-stack AdaLN prepass) must keep that prepass off while caching. Do not pass `--cache-dit-config` for native SGLang tuning unless you are intentionally using the diffusers backend flow. |
| **CFG Gating** | Per-request `--cfg-gate-step 0.5` (sampling param; also via API `extra_body`). `SGLANG_DIFFUSION_CFG_GATE_STEP` is the server-wide default (1.0 = off). | After the given fraction of denoising steps, reuses the cached cond-uncond residual instead of running the unconditional branch each step. | Up to ~2x on the gated tail of CFG models (skips one of two branches) | Lossy; no-op without classifier-free guidance or with `--enable-cfg-parallel`. Lower fractions gate earlier and drift more. |
@@ -373,7 +373,7 @@ For video, also match the captured frame and conditioning shape; `WxH` alone
does not prove replay.
For a repeated discovery sweep, use the benchmark/profile helper. This runs
lossless, extra-high, and high Eager/BCG ABBA pairs on one GPU set, then deletes the
lossless, lossless, and high Eager/BCG ABBA pairs on one GPU set, then deletes the
model group cache once:
```bash
@@ -391,8 +391,8 @@ sglang generate --model-path <MODEL> \
--perf-dump-path baseline.json --save-output
sglang generate --model-path <MODEL> \
--quality extra-high --prompt "..." --seed 42 \
--perf-dump-path quality-extra-high.json --save-output
--quality lossless --prompt "..." --seed 42 \
--perf-dump-path quality-lossless.json --save-output
sglang generate --model-path <MODEL> \
--quality high --prompt "..." --seed 42 \
@@ -400,7 +400,7 @@ sglang generate --model-path <MODEL> \
```
Keep every other flag fixed and compare the generated artifact as well as the
perf dumps. `high` must retain every fusion observed under `extra-high`. If the
perf dumps. `high` must retain every fusion observed under `lossless`. If the
model has no registered request-gated or high-only sites, either tier may be a
no-op.
@@ -510,7 +510,7 @@ Use these as first commands to benchmark, not as universal winners.
| Z-Image / Z-Image-Turbo | 1024x1024, runtime-default steps/guidance, 1 GPU | `--enable-torch-compile --warmup-mode request` | Keep base Z-Image separate from Turbo: base uses 50-step CFG defaults, Turbo uses 9-step zero-CFG defaults. Mainline has bf16-native Triton RMSNorm scale and tanh-residual fusions. |
| Wan2.2 A14B T2V/I2V | 1280x720, 81 frames | Nightly: `--num-gpus 4 --enable-cfg-parallel --ulysses-degree 2 --text-encoder-cpu-offload --pin-cpu-memory` | For lowest latency, also benchmark pure Ulysses on the same GPUs. |
| Wan2.2 TI2V 5B | 1280x720, 81 frames, 1 GPU | `--enable-torch-compile --warmup-mode request` | Keep the input image and motion prompt fixed when comparing sparse attention or Cache-DiT. |
| Wan2.1 / FastWan / TurboWan variants | 480p or 720p video, family defaults | Compare `--quality lossless`, `--quality extra-high`, and `--quality high`, then try `--enable-torch-compile --warmup-mode request`; add `--ulysses-degree` / CFG parallel only after measuring | `extra-high` and `high` mount the Wan FFN cublasLt/NVFP4 GELU epilogues and the Wan VAE RMSNorm+SiLU fast path when their guards pass; validate video quality against lossless. Current registry includes Wan2.1, FastWan2.1, FastWan2.2 TI2V, TurboWan2.1, TurboWan2.2 I2V, and Wan2.1-Fun InP. Use the compatibility matrix and benchmark presets before choosing topology. |
| Wan2.1 / FastWan / TurboWan variants | 480p or 720p video, family defaults | Compare `--quality lossless`, `--quality lossless`, and `--quality high`, then try `--enable-torch-compile --warmup-mode request`; add `--ulysses-degree` / CFG parallel only after measuring | `lossless` and `high` mount the Wan FFN cublasLt/NVFP4 GELU epilogues and the Wan VAE RMSNorm+SiLU fast path when their guards pass; validate video quality against lossless. Current registry includes Wan2.1, FastWan2.1, FastWan2.2 TI2V, TurboWan2.1, TurboWan2.2 I2V, and Wan2.1-Fun InP. Use the compatibility matrix and benchmark presets before choosing topology. |
| Cosmos3 Nano / Super | T2I: 1024x1024 with `--num-frames 1`; T2V/I2V: 480p/720p video | Start with `--performance-mode auto --warmup-mode request`; use `SGLANG_DISABLE_COSMOS3_GUARDRAILS=1` only for benchmark isolation, and compare compile separately | One checkpoint serves T2I/T2V/I2V. Mode is request-driven: `num_frames == 1` means T2I, `--image-path` means I2V. On GPUs with at least 120 GiB available, auto mode keeps the Cosmos3 DiT and VAE resident for every checkpoint in the family; a 1xH200 832x480x9f, 4-step eager ABBA reduced e2e from 1.576 to 0.428 seconds with exact output parity. Cosmos3 runs one DiT per pipeline, so component offload above that threshold only buys a DiT copy out to host memory and back per request -- it cost Cosmos3-Super 720p 81f T2V ~4s of ~115s on 2xH200. |
| Cosmos3 Edge / distilled Super | Edge T2I: 640x640, 35 steps, 1 GPU; distilled Super T2I: 640x640, fixed 4-step schedule, 4 GPUs | Start eager with `--performance-mode manual`; use `SGLANG_DISABLE_COSMOS3_GUARDRAILS=1` only for benchmark isolation | Edge is trained for 256p/480p shapes. Distilled checkpoints own their sigma schedule and force guidance 1.0; do not override steps or flow shift. Do not retry the closed experimental Cosmos BCG path without a new lifecycle design. |
| Ideogram 4 FP8/NVFP4 | 1024x1024, native preset defaults | `--enable-torch-compile --warmup-mode request` | Do not set `--num-inference-steps` or `--guidance-scale` directly unless you also update the Ideogram preset; sampling params derive them from `preset`. |
@@ -523,7 +523,7 @@ Use these as first commands to benchmark, not as universal winners.
| JoyAI-Image-Edit | 1024-class TI2I, 40 steps, guidance 4.0 | `--backend=sglang --num-gpus 2 --enable-cfg-parallel --ulysses-degree 1 --enable-torch-compile --warmup-mode request --dit-layerwise-offload false --dit-cpu-offload false` | Newly supported image-edit path. Keep the input image, prompt, seed, and output size fixed; 2-GPU CFG parallel is the validated H100 starting point. |
| FireRed-Image-Edit 1.0 / 1.1 | 1024x1024 image edit, 40 steps, guidance 4.0 | `--backend=sglang --num-gpus 2 --enable-cfg-parallel --ulysses-degree 1 --enable-torch-compile --warmup-mode request --dit-layerwise-offload false --dit-cpu-offload false` | Uses the native `QwenImageEditPlusPipeline` path. 2-GPU CFG parallel is the validated H100 starting point; benchmark 1.0 and 1.1 separately because checkpoint differences can change denoise latency. |
| Hunyuan3D-2 shape | Shape generation, 50 steps, guidance 5.0 | `--backend=sglang --enable-torch-compile --warmup-mode request --dit-layerwise-offload false --dit-cpu-offload false` | Focus on `Hunyuan3DShapeDenoisingStage`; keep mesh export/paint timings separate from denoise. |
| LingBot Video MoE 30B | 384x640, 17 frames, 12 steps for the current GPU case | `--model-path robbyant/lingbot-video-moe-30b-a3b --text-encoder-cpu-offload` | Native T2V path. Prompts are structured JSON captions, not raw free text; keep that contract when comparing latency or quality. Current main can mount the fused Triton RMSNorm path at `quality=extra-high` or `quality=high`; keep `lossless` as the reference. `--text-encoder-cpu-offload` targets memory-bound multi-GPU or small-VRAM cards; on a single large-VRAM GPU (e.g. 275 GB B300) the whole model stays resident (~73 GB peak), so dropping the flag removes H2D/D2H traffic and was 8% faster end to end (3.80s -> 3.49s, bit-identical). |
| LingBot Video MoE 30B | 384x640, 17 frames, 12 steps for the current GPU case | `--model-path robbyant/lingbot-video-moe-30b-a3b --text-encoder-cpu-offload` | Native T2V path. Prompts are structured JSON captions, not raw free text; keep that contract when comparing latency or quality. Current main can mount the fused Triton RMSNorm path at `quality=lossless` or `quality=high`; keep `exact` as the reference. `--text-encoder-cpu-offload` targets memory-bound multi-GPU or small-VRAM cards; on a single large-VRAM GPU (e.g. 275 GB B300) the whole model stays resident (~73 GB peak), so dropping the flag removes H2D/D2H traffic and was 8% faster end to end (3.80s -> 3.49s, bit-identical). |
| MOVA / Helios / LingBot World | Start from the family's cookbook page — it pins the GPU count and parallelism these realtime paths need (e.g. LingBot World is served with `--num-gpus 4 --ulysses-degree 4`, LingBot World 2.0 with 8) | `--enable-torch-compile --warmup-mode request`; pin offload and topology flags explicitly | These video/realtime families have model-specific stages and condition handling. Realtime keeps up with playback only at the documented GPU count, so do not tune down from it by accident. For LingBot World causal serving, keep `--kv-cache-quant off` as the exact cache baseline before testing INT4/INT2. |
## Historical PR Watchlist
@@ -99,8 +99,8 @@ class SGLDiffusionServerAPI:
seed: Random seed for reproducible generation
enable_teacache: Enable TEA cache acceleration
response_format: Response format ("b64_json" or "url")
quality: Request optimization tier ("auto", "lossless",
"extra-high", "high") - only for generation
quality: Request optimization tier ("auto", "exact",
"lossless", "high") - only for generation
style: Image style ("vivid" or "natural") - only for generation
background: Background type ("auto", "transparent", "opaque")
output_format: Output format ("png", "jpeg", "webp")
@@ -342,7 +342,7 @@ class FastH3PipelineConfig(MiniMaxH3PipelineConfig):
raise ValueError(
'quality="high" is audited only for the base MiniMax-H3 50-step '
"4xH200 deployment; the FastH3 4-step distilled checkpoint has no "
'audited high-quality deployment. Use quality="lossless".'
'audited high-quality deployment. Use quality="exact".'
)
def validate_server_args(self, server_args) -> None:
@@ -23,7 +23,7 @@ class VDNH3PipelineConfig(MiniMaxH3PipelineConfig):
raise ValueError(
'quality="high" is audited only for the base MiniMax-H3 50-step '
"4xH200 deployment; the VDN-H3 8-step hybrid checkpoint has no "
'audited high-quality deployment. Use quality="lossless".'
'audited high-quality deployment. Use quality="exact".'
)
def validate_server_args(self, server_args) -> None:
@@ -53,12 +53,15 @@ def generate_request_id() -> str:
return str(uuid.uuid4())
# Validated request-level quality levels, ordered from the strictest numerical
# contract to the broadest optimization set. "lossless" keeps the exact
# reference path; "extra-high" adds only request-gated kernel fusions; "high"
# is cumulative and may also enable model-owned approximate optimizations.
QUALITY_LEVELS: tuple[str, ...] = ("lossless", "extra-high", "high")
KERNEL_FUSION_QUALITY_LEVELS = frozenset({"extra-high", "high"})
# Request-level quality levels, ordered from the strictest numerical contract to
# the broadest optimization set; each level includes the fast paths of the ones
# before it:
# - "exact": bit-identical to the reference path in the same environment
# - "lossless": the same math; fast paths may change rounding order or position
# - "high": may also change what is computed (precision, sparsity, reuse)
QUALITY_LEVELS: tuple[str, ...] = ("exact", "lossless", "high")
# "extra-high" is the pre-rename name of "lossless"
QUALITY_ALIASES: dict[str, str] = {"extra-high": "lossless"}
# libx264 presets, fastest first: each spends more encode time for a smaller
# file at the same CRF.
X264_PRESETS: tuple[str, ...] = (
@@ -126,9 +129,20 @@ def resolve_skip_softmax_params(
return SkipSoftmaxParams(float(threshold), start_step)
def quality_allows_kernel_fusions(quality: str) -> bool:
"""Return whether a quality level includes request-gated kernel fusions."""
return quality in KERNEL_FUSION_QUALITY_LEVELS
def normalize_quality(quality: str) -> str:
"""Return the canonical quality level for ``quality``, resolving aliases."""
level = QUALITY_ALIASES.get(quality, quality)
if level not in QUALITY_LEVELS:
raise ValueError(
f"quality must be one of {[*QUALITY_LEVELS, *QUALITY_ALIASES]}, "
f"got {quality!r}"
)
return level
def quality_allows(quality: str, tier: str) -> bool:
"""Whether a request at ``quality`` may run a fast path of class ``tier``."""
return QUALITY_LEVELS.index(quality) >= QUALITY_LEVELS.index(tier)
def _sanitize_filename(name: str, replacement: str = "_", max_length: int = 150) -> str:
@@ -248,18 +262,20 @@ class SamplingParams:
output_compression: int | None = None
# MP4 encode speed; None keeps the server's default preset
x264_preset: str | None = None
# Model-owned, request-scoped quality level.
# Model-owned, request-scoped quality level (see QUALITY_LEVELS).
#
# - "lossless" (default): the exact reference path. Output is expected to
# be bit-identical to the HF reference implementation and to pass the
# CI golden/ground-truth comparisons.
# - "extra-high": add only validated kernel fusions. These may change
# half-precision rounding order, so output is not bit-exact versus the
# reference, but this tier does not itself enable sparse or approximate
# optimizations.
# - "high": include every "extra-high" fusion and allow model-owned
# approximate optimizations such as sparse computation or feature
# caching. These paths require model-specific quality validation.
# - "exact": the reference path, plus the fast paths that reproduce its
# rounding exactly. Output is bit-identical to it in the same
# environment; ask for this when a run has to be reproducible or when
# bisecting a numerical bug.
# - "lossless" (default): also mount the fast paths that keep the
# reference math and the precision of every operand and accumulator,
# and only move where the rounding happens. Output matches the
# reference to rounding error rather than bit for bit, by less than the
# same path already moves between two legal environments.
# - "high": also allow model-owned approximate optimizations (lower
# precision, sparse computation, feature caching), which need
# model-specific quality validation.
#
# It intentionally participates in the dynamic-batch signature.
quality: str = "lossless"
@@ -604,10 +620,7 @@ class SamplingParams:
f"prompt_path must be a txt file, got {self.prompt_path!r}"
)
if self.quality not in QUALITY_LEVELS:
raise ValueError(
f"quality must be one of {list(QUALITY_LEVELS)}, got {self.quality!r}"
)
self.quality = normalize_quality(self.quality)
resolve_skip_softmax_params(self.skip_softmax_params)
@@ -1257,16 +1270,17 @@ class SamplingParams:
add_argument(
"--quality",
type=str,
choices=list(QUALITY_LEVELS),
choices=[*QUALITY_LEVELS, *QUALITY_ALIASES],
help=(
"Request-level quality: 'lossless' (default) keeps the exact "
"reference path, bit-exact against the reference "
"implementation; 'extra-high' adds only request-gated kernel "
"fusions and does not itself enable sparse or approximate "
"optimization; 'high' includes every extra-high fusion and "
"may also enable "
"model-owned approximate paths. Support and validated "
"deployment constraints are model-specific."
"Request-level quality: 'lossless' (default) runs every fast "
"path that keeps the reference math and operand precision and "
"only moves the rounding, so output matches the reference to "
"rounding error; 'exact' restricts it to the fast paths that "
"reproduce the reference's rounding exactly, giving "
"bit-identical output in the same environment; 'high' may "
"also enable model-owned approximate paths. 'extra-high' is "
"accepted as the former name of 'lossless'. Support and "
"validated deployment constraints are model-specific."
),
)
add_argument(
@@ -102,6 +102,7 @@ from sglang.multimodal_gen.runtime.utils.logging_utils import (
configure_logger,
init_logger,
)
from sglang.multimodal_gen.runtime.utils.numerics_policy import apply_numerics_policy
from sglang.multimodal_gen.runtime.utils.perf_logger import (
PerformanceLogger,
capture_memory_snapshot,
@@ -384,6 +385,12 @@ class GPUWorker(GPUWorkerPostTrainingMixin):
if not current_platform.is_mps():
current_platform.set_device(current_platform.get_device(self.local_rank))
self._cap_device_memory_for_tests()
apply_numerics_policy(
allow_cudnn_tf32=self.server_args.allow_cudnn_tf32,
allow_bf16_reduced_precision_reduction=(
self.server_args.allow_bf16_reduced_precision_reduction
),
)
# num_gpus is the total world size across every node; the co-located,
# CPU-contending worker count on THIS host is num_gpus // nnodes.
local_num_gpus = self.server_args.num_gpus // self.server_args.nnodes
@@ -175,7 +175,7 @@ def _flux_norm_modulate(
Priority: (1) the bit-exact single-kernel LN+modulate -- lossless, so it
needs no quality gate and also supersedes the request-gated affine
fold wherever it verifies; (2) when the site is mounted
(``quality="extra-high"`` or ``"high"``) and the bit-exact kernel is
(``quality="lossless"`` or ``"high"``) and the bit-exact kernel is
unavailable, the modulate folded into the LN affine (one aten kernel; not
bit-exact);
(3) affine-free LayerNorm + the bit-exact fused modulate.
@@ -409,7 +409,7 @@ class FluxGELU(nn.Module):
prefix=f"{prefix}.proj" if prefix else "proj",
)
self.gelu = nn.GELU(approximate="tanh")
# extra-high/high fusion site: up-proj GEMM + tanh-GELU in the cublasLt
# lossless/high fusion site: up-proj GEMM + tanh-GELU in the cublasLt
# epilogue. Off by default; mounted per batch by the denoising stage.
mark_fused_gelu_site(self, "proj")
@@ -429,7 +429,7 @@ class FluxFusedGELUProj(nn.Module):
``approximate="tanh"`` that keeps the ``net.0.proj`` parameter path. The
default path is the bit-exact reference (plain Linear + tanh-GELU); the
cublasLt GELU epilogue is mounted per batch by the denoising stage for
requests with ``quality="extra-high"`` or ``quality="high"`` only.
requests with ``quality="lossless"`` or ``quality="high"`` only.
"""
def __init__(self, proj: nn.Linear):
@@ -877,7 +877,7 @@ class FluxSingleTransformerBlock(nn.Module):
prefix=f"{prefix}.proj_mlp" if prefix else "proj_mlp",
)
self.act_mlp = nn.GELU(approximate="tanh")
# extra-high/high fusion site: proj_mlp GEMM + tanh-GELU in the
# lossless/high fusion site: proj_mlp GEMM + tanh-GELU in the
# cublasLt epilogue (mounted per batch by the denoising stage).
mark_fused_gelu_site(self, "proj_mlp")
proj_out_cls = (
@@ -1044,7 +1044,7 @@ class FluxTransformerBlock(nn.Module):
self.norm2 = LayerNorm(dim, eps=1e-6, elementwise_affine=False)
self.norm2_context = LayerNorm(dim, eps=1e-6, elementwise_affine=False)
# extra-high/high site: norm2/norm2_context modulate folds into the
# lossless/high site: norm2/norm2_context modulate folds into the
# LN affine when mounted.
mark_fused_ln_modulate_site(self)
@@ -1099,7 +1099,7 @@ class FluxTransformerBlock(nn.Module):
activation_fn="gelu-approximate",
)
# Re-home each FF's tanh-GELU up-projection onto a marked
# extra-high/high fusion site (bit-exact reference by default).
# lossless/high fusion site (bit-exact reference by default).
self.ff.net[0] = FluxFusedGELUProj(self.ff.net[0].proj)
self.ff_context.net[0] = FluxFusedGELUProj(self.ff_context.net[0].proj)
@@ -448,7 +448,7 @@ class GlmImageGELU(nn.Module):
quant_config=quant_config,
prefix=f"{prefix}.proj" if prefix else "proj",
)
# extra-high/high fusion site: up-proj GEMM + tanh-GELU in cublasLt
# lossless/high fusion site: up-proj GEMM + tanh-GELU in cublasLt
# epilogue. Off by default; mounted per batch by the denoising stage.
mark_fused_gelu_site(self, "proj")
@@ -399,7 +399,7 @@ def _norm_scale(
norm: Ideogram4RMSNorm,
enable_fused: bool,
) -> torch.Tensor:
"""``RMSNorm(x) * (1 + scale)``, fused at extra-high or high quality."""
"""``RMSNorm(x) * (1 + scale)``, fused at high quality."""
if enable_fused:
y = fused_rmsnorm_scale(
x,
@@ -433,7 +433,7 @@ def _gate_residual(
norm: Ideogram4RMSNorm,
enable_fused: bool,
) -> torch.Tensor:
"""``residual + tanh(gate) * RMSNorm(x)``, fused at extra-high or high."""
"""``residual + tanh(gate) * RMSNorm(x)``, fused at high quality."""
if enable_fused:
y = fused_rmsnorm_tanh_residual(
x,
@@ -489,7 +489,7 @@ class Ideogram4TransformerBlock(nn.Module):
self.ffn_norm1 = Ideogram4RMSNorm(hidden_size, eps=norm_eps)
self.attention_norm2 = Ideogram4RMSNorm(hidden_size, eps=norm_eps)
self.ffn_norm2 = Ideogram4RMSNorm(hidden_size, eps=norm_eps)
# extra-high/high fusion sites: each RMSNorm modulate/gate chain
# high-tier fusion sites: each RMSNorm modulate/gate chain
# collapses into one Triton kernel (Z-Image bf16-native suite). Off by
# default (bit-exact reference path); mounted per batch by the
# denoising stage.
@@ -114,7 +114,7 @@ def _lingbot_gated_residual(
Uses the request-gated per-token ``residual_gate_add`` fast path when the
block's quality-gated site is enabled; otherwise the reference FP32
multiply form (bit-exact for ``quality="lossless"``).
multiply form (bit-exact for ``quality="exact"``).
"""
fused = try_lingbot_video_gated_residual(block, residual, update, gate)
if fused is not None:
@@ -324,7 +324,7 @@ class _LongCatFFN(nn.Module):
]
)
self.act = nn.GELU(approximate="tanh")
# extra-high/high site: up-proj GEMM + tanh-GELU epilogue. Off by
# lossless/high site: up-proj GEMM + tanh-GELU epilogue. Off by
# default; the denoising stage mounts it per batch. The ModuleDict holds
# `proj` in _modules, so getattr resolves it for the fusion helper.
mark_fused_gelu_site(self.net[0], "proj")
@@ -613,7 +613,7 @@ class _SingleTransformerBlock(nn.Module):
prefix=f"{prefix}.proj_mlp",
)
self.act_mlp = nn.GELU(approximate="tanh")
# extra-high/high site: proj_mlp GEMM + tanh-GELU epilogue,
# lossless/high site: proj_mlp GEMM + tanh-GELU epilogue,
# mounted per batch by the denoising stage; off (bit-exact) by default.
mark_fused_gelu_site(self, "proj_mlp")
# proj_out: RowParallelLinear reduces sharded [attn | mlp] concat via
@@ -12,7 +12,7 @@ import torch.nn.functional as F
from sglang.kernels.ops.diffusion import (
BitExactFusionGate,
can_use_fused_rmsnorm_scale_shift,
can_use_fused_rmsnorm_modulation,
can_use_linear_gelu,
can_use_ltx2_qknorm_split_rope_cuda,
can_use_modulate_scale_shift_cuda,
@@ -204,14 +204,12 @@ def _ltx2_rms_norm_modulate(
Folds the weightless RMSNorm and the modulate into one kernel when the
request-gated fusion is mounted on ``block`` and the per-call guard
passes; otherwise the verbatim eager reference chain (the ``lossless``
passes; otherwise the verbatim eager reference chain (the ``exact``
default). The fused kernel is not bit-exact (<=1 bf16 ULP) so it is gated
on the request-scoped mount rather than a runtime self-check.
"""
if (
ltx2_rms_norm_modulate_active(block)
and x.is_cuda
and can_use_fused_rmsnorm_scale_shift(x.dtype, x.shape[-1])
if ltx2_rms_norm_modulate_active(block) and can_use_fused_rmsnorm_modulation(
x, scale, shift
):
return fused_ltx2_rms_norm_modulate(x, scale, shift, eps)
normed = rms_norm(x, eps)
@@ -999,8 +999,8 @@ class QwenImageCrossAttention(nn.Module):
prefix=f"{prefix}.to_added_qkv",
)
if self._unquantized_added_qkv_is_packed:
# Packing changes BF16 GEMM reduction association. Keep it
# off for lossless and mount it at extra-high or high.
# Packing changes BF16 GEMM reduction association, so it
# is off at exact and mounts at lossless or high.
mark_qwen_image_added_qkv_site(self)
else:
self.add_q_proj = ColumnParallelLinear(
@@ -8,8 +8,8 @@ decoder module family (``ResnetBlock2D`` GroupNorm+SiLU chains,
All rewrites are mathematically exact re-associations of the original
operators. Wrappers are installed once at VAE load and dispatch on a
decode-scoped :class:`VaeFastPathGate`: ``quality="extra-high"`` and
``quality="high"`` run the fast paths, while the ``"lossless"`` default runs
decode-scoped :class:`VaeFastPathGate`: ``quality="lossless"`` and
``quality="high"`` run the fast paths, while the ``"exact"`` default runs
the original module path bit-for-bit.
- channels_last: run the decoder in NHWC so cuDNN convs skip the transpose
@@ -4,9 +4,9 @@ Qwen-Image VAE, which is the Wan 2.1 VAE under other class names).
Fuses every decoder ``WanRMS_norm -> SiLU`` chain into one Triton kernel on
the channels_last_3d layout. Wrappers are installed once at VAE load and
dispatch on a decode-scoped :class:`VaeFastPathGate`: ``quality="extra-high"``
dispatch on a decode-scoped :class:`VaeFastPathGate`: ``quality="lossless"``
and ``quality="high"`` run the fused kernel (not bitwise-identical to aten,
hence gated). The ``"lossless"`` path preserves aten's normalization reduction
hence gated). The ``"exact"`` path preserves aten's normalization reduction
and fuses its FP32 post-ops after first-sight exactness verification.
Install is all-or-nothing and fail-closed.
"""
@@ -160,7 +160,7 @@ class GatedChannelsLastUpsample(nn.Module):
# The predicate admits exactly the inputs on which aten itself would
# run its NHWC kernel and return a dense channels_last tensor, so the
# Triton gather is a layout- and value-identical replacement: it runs
# on the lossless path too (the gate only controls the stride
# on the exact path too (the gate only controls the stride
# canonicalisation above).
if up.size is None and can_use_nearest_upsample_nhwc(
x, up.scale_factor, up.mode
@@ -11,7 +11,7 @@ import torch
import torch.nn as nn
from sglang.multimodal_gen.configs.sample.sampling_params import (
quality_allows_kernel_fusions,
quality_allows,
)
from sglang.multimodal_gen.runtime.distributed import (
get_decode_parallel_world_size,
@@ -334,9 +334,11 @@ class DecodingStage(PipelineStage):
assert vae is not None
self.vae = vae
# The decoder fast paths re-associate the reference operators
# without lowering any precision, so they belong to "lossless".
with use_vae_fast_path(
vae,
quality_allows_kernel_fusions(batch.sampling_params.quality),
quality_allows(batch.sampling_params.quality, "lossless"),
):
frames = self.decode(batch.latents, server_args, vae_dtype=vae_dtype)
@@ -56,7 +56,7 @@ from sglang.multimodal_gen.configs.pipeline_configs.flux import (
)
from sglang.multimodal_gen.configs.pipeline_configs.zimage import ZImagePipelineConfig
from sglang.multimodal_gen.configs.sample.sampling_params import (
quality_allows_kernel_fusions,
quality_allows,
resolve_skip_softmax_params,
)
from sglang.multimodal_gen.configs.task_type import get_request_task_type
@@ -177,70 +177,95 @@ from sglang.multimodal_gen.runtime.utils.torch_compile import (
logger = init_logger(__name__)
# Request-gated DiT fusions and the lowest quality level that may mount each
# one. A fusion belongs to "lossless" when it keeps the reference math and the
# precision of every operand and accumulator, changing only where or in which
# order the rounding happens; one that lowers a precision or quantizes belongs
# to "high". No entry is tier "exact": the default must keep running the same
# kernels it runs today, so a by-construction bit-exact fast path has to come
# with a BitExactFusionGate before it can move there.
_QUALITY_FUSION_HANDLERS: tuple[
tuple[str, Callable[[nn.Module], bool], Callable[[nn.Module], None]], ...
tuple[str, str, Callable[[nn.Module], bool], Callable[[nn.Module], None]], ...
] = (
(
# quantizes FC2's input before the reference BF16 intermediate exists
"high",
"FLUX.2 NVFP4 FC1+SwiGLU+quant",
mount_flux2_nvfp4_swiglu_quant,
unmount_flux2_nvfp4_swiglu_quant,
),
(
"lossless",
"fused linear+GELU (cublasLt epilogue)",
mount_fused_linear_gelu,
unmount_fused_linear_gelu,
),
(
"lossless",
"Wan NVFP4 fused bias+GELU",
mount_nvfp4_bias_gelu,
unmount_nvfp4_bias_gelu,
),
(
"lossless",
"Qwen-Image fused added-QKV",
mount_qwen_image_added_qkv,
unmount_qwen_image_added_qkv,
),
(
"lossless",
"fused LN+modulate (affine folding)",
mount_fused_ln_modulate,
unmount_fused_ln_modulate,
),
(
"lossless",
"LTX-2 Hopper QKNorm+split-RoPE",
mount_ltx2_qknorm_split_rope,
unmount_ltx2_qknorm_split_rope,
),
(
"lossless",
"LTX-2 fused RMSNorm+modulate",
mount_ltx2_rms_norm_modulate,
unmount_ltx2_rms_norm_modulate,
),
(
# Ideogram's reference keeps the norm statistics in fp32; the
# BF16-native kernel rounds them, which lowers accumulation precision
"high",
"fused gate RMSNorm (BF16-native Triton)",
mount_fused_gate_rmsnorm,
unmount_fused_gate_rmsnorm,
),
(
"lossless",
"HunyuanVideo strided QK RMSNorm",
mount_hunyuan_qknorm,
unmount_hunyuan_qknorm,
),
(
"lossless",
"LingBot Video fused RMSNorm",
mount_lingbot_video_rmsnorm,
unmount_lingbot_video_rmsnorm,
),
(
"lossless",
"LingBot Video per-token gated residual",
mount_lingbot_video_gated_residual,
unmount_lingbot_video_gated_residual,
),
(
"lossless",
"Helios per-token gated residual",
mount_helios_gated_residual,
unmount_helios_gated_residual,
),
(
# the first attention GEMM takes BF16 inputs where the reference
# promotes them to FP32
"high",
"SANA-Video BF16-input linear attention",
mount_sana_video_linear_attention,
unmount_sana_video_linear_attention,
@@ -359,11 +384,12 @@ class DenoisingStage(PipelineStage, RolloutDenoisingMixin):
self._cache_dit_request_overrides: dict[str, Any] = {}
# Overrides key the mounted hooks were built from; None when unmounted.
self._cache_dit_active_key: tuple | None = None
# Whether request-scoped extra-high-or-higher fusions are mounted.
self._quality_fusions_mounted = False
# The quality level whose request-scoped fusions are mounted.
self._mounted_quality = "exact"
self._torch_compile_registry = CompiledModuleRegistry()
# Breakable CUDA graph runners, one per transformer module (lazy).
self._bcg_runners: dict[int, Any] = {}
# Breakable CUDA graph runners, lazily created per (module, quality
# level); see _maybe_get_bcg_runner for why the level is in the key.
self._bcg_runners: dict[tuple[int, str], Any] = {}
hidden_size = self.server_args.pipeline_config.dit_config.hidden_size
num_attention_heads = (
@@ -763,40 +789,32 @@ class DenoisingStage(PipelineStage, RolloutDenoisingMixin):
def _maybe_toggle_quality_fusions(self, batch: Req) -> None:
"""Mount/unmount request-gated kernel fusions for this batch.
These fusions are numerically equivalent only at half-precision
rounding level (not bit-exact), so they are mounted for both
``quality="extra-high"`` and ``quality="high"``. The ``"lossless"``
default runs the reference path bit-for-bit. ``quality`` participates
in the dynamic-batch signature, making this transition safe at the
batch boundary. Mounting is all-or-nothing per transformer and fusion
Each fusion declares the lowest quality level that may mount it (see
``_QUALITY_FUSION_HANDLERS``), so a request mounts the fusions of its
own tier and of every stricter one; ``"exact"`` mounts none of them
and runs the reference path bit-for-bit. ``quality`` participates in
the dynamic-batch signature, making this transition safe at the batch
boundary. Mounting is all-or-nothing per transformer and fusion
family; models without marked sites are no-ops.
Under breakable CUDA graphs the mounted set is baked into whatever was
captured, which is why the runner is keyed by level as well as module
(see :meth:`_maybe_get_bcg_runner`): changing level here cannot
silently replay another level's kernels.
"""
quality = getattr(batch.sampling_params, "quality", "lossless")
want = quality_allows_kernel_fusions(quality)
if want == self._quality_fusions_mounted:
if quality == self._mounted_quality:
return
mounted_fusions: set[str] = set()
for transformer in filter(None, [self.transformer, self.transformer_2]):
for description, mount, unmount in _QUALITY_FUSION_HANDLERS:
if want:
for tier, description, mount, unmount in _QUALITY_FUSION_HANDLERS:
if quality_allows(quality, tier):
if mount(transformer):
mounted_fusions.add(description)
else:
unmount(transformer)
if want and mounted_fusions and self.server_args.enable_breakable_cuda_graph:
for transformer in filter(None, [self.transformer, self.transformer_2]):
for _, _, unmount in _QUALITY_FUSION_HANDLERS:
unmount(transformer)
descriptions = ", ".join(sorted(mounted_fusions))
raise ValueError(
f"quality={quality!r} cannot be used with breakable CUDA graphs for "
f"this model because its request-scoped DiT fusions "
f"({descriptions}) do not match the lossless warmup graphs. "
"Disable breakable CUDA graphs or use quality='lossless'."
)
self._quality_fusions_mounted = want
self._mounted_quality = quality
for description in sorted(mounted_fusions):
logger.debug("Mounted %s for quality=%s", description, quality)
@@ -2589,19 +2607,38 @@ class DenoisingStage(PipelineStage, RolloutDenoisingMixin):
return None
if not isinstance(current_model, nn.Module):
return None
key = id(current_model)
# A captured graph bakes in whichever fusions were mounted when it was
# recorded, and the capture signature only covers the tensors -- so a
# runner is per (module, quality level). Warmup runs at the default
# level and captures its graphs there; a request at another level
# finds an empty runner and the eager fallback runs the fusion set it
# actually asked for. Sharing one runner would replay the warmup
# level's kernels under a different level's name.
key = (id(current_model), self._mounted_quality)
runner = self._bcg_runners.get(key)
if runner is None:
from sglang.multimodal_gen.runtime.breakable_cuda_graph.runner import (
DiffusionBreakableCudaGraphRunner,
)
# Another level already has a runner, so this one arrived after
# warmup and will never hold a captured graph.
served_another_level = any(
model_id == id(current_model) for model_id, _ in self._bcg_runners
)
# DenoisingStage can switch between transformer and transformer_2;
# each module owns separate graph state and static input buffers.
runner = DiffusionBreakableCudaGraphRunner(
current_model, get_local_torch_device()
)
self._bcg_runners[key] = runner
if served_another_level:
logger.info_once(
"quality=%s has no captured breakable CUDA graphs (warmup "
"captures the server's default level), so its requests run "
"eager.",
self._mounted_quality,
)
return runner
def prepare_sta_param(self, batch: Req, server_args: ServerArgs):
@@ -18,7 +18,7 @@ from sglang.kernels.ops.diffusion import (
unmount_helios_gated_residual,
)
from sglang.multimodal_gen.configs.sample.sampling_params import (
quality_allows_kernel_fusions,
quality_allows,
)
from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType
from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context
@@ -116,11 +116,12 @@ class HeliosChunkedDenoisingStage(PipelineStage):
self._quality_fusions_mounted = False
def _maybe_toggle_quality_fusions(self, batch: Req) -> None:
# Mount the request-scoped Helios gated-residual fast path for
# quality="extra-high"/"high"; "lossless" keeps the reference
# FP32-multiply form bit-for-bit.
# The gated-residual fusion rounds the FP32 gate and update to BF16
# before multiplying, which moves a rounding without lowering the
# reference's own operand precision: tier "lossless". The "exact"
# default keeps the reference FP32-multiply form bit-for-bit.
quality = getattr(batch.sampling_params, "quality", "lossless")
want = quality_allows_kernel_fusions(quality)
want = quality_allows(quality, "lossless")
if want == self._quality_fusions_mounted:
return
self._quality_fusions_mounted = want
@@ -57,8 +57,8 @@ MINIMAX_H3_DEFAULT_BRANCHES: tuple = ({"name": "cond_1"},)
# Audited 4xH200 T2VA Cache-DiT parameters for quality="high":
# (warmup steps, residual-difference threshold, max consecutive cached steps).
# Measured SSIM 0.931 / PSNR 28.16 dB against quality="lossless" on the
# validated workload. quality="lossless" (the default) uses no Cache-DiT
# Measured SSIM 0.931 / PSNR 28.16 dB against quality="exact" on the
# validated workload. quality="exact" (the default) uses no Cache-DiT
# configuration at all. Process-wide SGLANG_CACHE_DIT_* environment controls
# remain available for manual experiments and are independent of this field.
MINIMAX_H3_HIGH_QUALITY_CACHE_DIT_CONFIG: tuple[int, float, int] = (4, 0.04, 1)
@@ -455,7 +455,7 @@ class MiniMaxH3DenoisingStage(DenoisingStage):
scheduler=None,
pipeline=pipeline,
)
self._minimax_h3_quality = "lossless"
self._minimax_h3_quality = "exact"
self._minimax_h3_cache_mode: str | None = None
def _owns_compile_warmup_lifecycle(self) -> bool:
@@ -463,7 +463,7 @@ class MiniMaxH3DenoisingStage(DenoisingStage):
def _cache_dit_requested(self) -> bool:
return (
getattr(self, "_minimax_h3_quality", "lossless") == "high"
getattr(self, "_minimax_h3_quality", "exact") == "high"
or super()._cache_dit_requested()
)
@@ -592,7 +592,7 @@ class MiniMaxH3DenoisingStage(DenoisingStage):
def _cache_dit_scm_masks(
self, primary_num_steps: int, secondary_num_steps: int | None = None
) -> tuple[str, str, list[int] | None, list[int] | None]:
if getattr(self, "_minimax_h3_quality", "lossless") == "high":
if getattr(self, "_minimax_h3_quality", "exact") == "high":
return "none", "dynamic", None, None
return super()._cache_dit_scm_masks(primary_num_steps, secondary_num_steps)
@@ -604,7 +604,7 @@ class MiniMaxH3DenoisingStage(DenoisingStage):
*,
secondary: bool = False,
) -> CacheDitConfig:
if secondary or getattr(self, "_minimax_h3_quality", "lossless") != "high":
if secondary or getattr(self, "_minimax_h3_quality", "exact") != "high":
return super()._build_cache_dit_config(
num_inference_steps,
steps_computation_mask,
@@ -841,8 +841,8 @@ class CudaPlatformBase(Platform):
"""Install the quality-gated FLUX.2 / AutoencoderKL / Wan / Qwen-Image
VAE decoder fast paths.
Requests with quality="extra-high" or "high" run the fast paths; the
"lossless" default runs the original module path bit-for-bit. See
Requests with quality="lossless" or "high" run the fast paths; the
"exact" default runs the original module path bit-for-bit. See
flux2_vae_cuda_opt and wan_vae_cuda_opt for details.
"""
try:
@@ -483,6 +483,12 @@ class ServerArgs(DisaggServerArgsMixin):
# stay eager). Mutually exclusive with --enable-torch-compile and
# Cache-DiT; BCG takes priority when more than one is requested.
#
# PyTorch's two approximate numerics defaults, set explicitly per worker
# (see runtime/utils/numerics_policy.py). They are process-global, so they
# cannot follow a request's quality level; the defaults keep PyTorch's own
# values so the exact tier's speed is unchanged.
allow_cudnn_tf32: bool = True
allow_bf16_reduced_precision_reduction: bool = True
# BCG graphs are resolution-specific, so --warmup-resolutions is required
# when BCG is enabled: every requested resolution is captured at warmup so
# serving never triggers a fresh capture.
@@ -2445,6 +2451,23 @@ class ServerArgs(DisaggServerArgsMixin):
default=ServerArgs.offload_during_compile,
help="Offload components during the torch.compile warmup (the DiT layerwise) so max-autotune fits on tighter-memory GPUs, then restore the configured residency for serving. Skipped when the DiT is already layerwise-offloaded, or under cache-dit / FSDP.",
)
parser.add_argument(
"--allow-cudnn-tf32",
action=StoreBoolean,
default=ServerArgs.allow_cudnn_tf32,
help="Let cuDNN run fp32 convolutions on TF32 tensor cores, which "
"truncates their inputs to 10 mantissa bits (PyTorch's own "
"default). Affects the fp32 VAE decoders; pass false together "
"with --allow-bf16-reduced-precision-reduction false for "
"reference-precision fp32 and bf16 math.",
)
parser.add_argument(
"--allow-bf16-reduced-precision-reduction",
action=StoreBoolean,
default=ServerArgs.allow_bf16_reduced_precision_reduction,
help="Let a bf16 GEMM accumulate its split-K partials below fp32 "
"(PyTorch's own default). See --allow-cudnn-tf32.",
)
parser.add_argument(
"--enable-breakable-cuda-graph",
action=StoreBoolean,
@@ -0,0 +1,65 @@
# SPDX-License-Identifier: Apache-2.0
"""Explicit control of the two approximate numerics PyTorch enables by default.
``torch.backends.cudnn.allow_tf32`` truncates fp32 convolution inputs to 10
mantissa bits, and
``torch.backends.cuda.matmul.allow_bf16_reduced_precision_reduction`` lets a
bf16 GEMM accumulate split-K partials below fp32. Both change what is
computed, so both sit at the ``high`` tier of ``QUALITY_LEVELS`` while the
runtime serves ``exact`` and ``lossless`` requests through the same process.
They cannot follow the request: they are process-global and a server mixes
tiers across concurrent batches. The runtime therefore sets them once per
worker from the server args and states the result, so a tier's numerical
contract is auditable instead of inherited from whichever PyTorch build is
installed.
The defaults keep PyTorch's own values. Turning TF32 off makes fp32
convolutions -- the VAE decoders that default to ``vae_precision="fp32"`` --
run on fp32 CUDA cores rather than tensor cores, so flipping the default would
slow the exact tier down. A deployment that needs the strict contract sets
both to false and pays that cost knowingly.
"""
from __future__ import annotations
import torch
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
logger = init_logger(__name__)
def apply_numerics_policy(
*, allow_cudnn_tf32: bool, allow_bf16_reduced_precision_reduction: bool
) -> None:
"""Set both approximate-numerics switches and log what this worker runs."""
torch.backends.cudnn.allow_tf32 = allow_cudnn_tf32
torch.backends.cuda.matmul.allow_bf16_reduced_precision_reduction = (
allow_bf16_reduced_precision_reduction
)
approximate = [
name
for name, enabled in (
("cudnn TF32", allow_cudnn_tf32),
(
"bf16 reduced-precision reduction",
allow_bf16_reduced_precision_reduction,
),
)
if enabled
]
if approximate:
logger.info_once(
"Numerics policy: %s enabled. These lower precision below the "
"reference, so quality='exact' is bit-reproducible on this server "
"but not free of approximate library defaults. Pass "
"--allow-cudnn-tf32 false --allow-bf16-reduced-precision-reduction "
"false for the strict contract (slower fp32 convolutions).",
" and ".join(approximate),
)
else:
logger.info_once(
"Numerics policy: cudnn TF32 and bf16 reduced-precision reduction "
"are both disabled; fp32 and bf16 math keeps reference precision."
)
@@ -0,0 +1,206 @@
# SPDX-License-Identifier: Apache-2.0
"""Admission checks for a fast path that claims the ``lossless`` tier.
A ``lossless``-tier fast path keeps the reference math and the precision of
every operand and accumulator; only the place or the order of the rounding
moves. That claim is not provable from a single tolerance, so these helpers
implement the checks a fusion has to pass before its tier is believed, and
replace the per-case thresholds each fusion test used to invent:
* :func:`assert_error_no_worse_than_reference` -- the operator-level gate. Both
the current implementation and the candidate are scored against an fp64
evaluation of the same math; the candidate may not be more than
``max_error_ratio`` times worse on RMS or max error, and its error may not
carry a systematic sign.
* :func:`assert_special_values_match` -- NaN, Inf, fully-masked rows and
non-contiguous inputs keep their current behavior.
* :func:`discrete_flip_rate` -- when a quantizer, top-k or threshold reads the
output, the quantity that matters is how often the discrete result changes,
not the size of the continuous error.
Two gates are deliberately not automated here, because they are properties of
the change rather than of one tensor comparison:
* the static conditions (accumulation at fp32 or better, no operand or
intermediate precision lowered, an approximate transcendental within 0.1 ulp
of the output dtype, stability no worse than the reference: two-pass or
Welford variance, max-subtracted softmax). A fast path that breaks one of
them is approximate by construction, and no measurement can admit it.
* the end-to-end drift band -- the difference against the exact tier has to sit
inside the spread the same reference path already shows across two legal
environments (H100 against H200, TP1 against TP2, two cuBLAS versions). That
needs a server run, so it belongs to the model's consistency suite.
Feed the operator-level gate activations captured from the model, not only
random tensors: DiT activations carry outlier channels that decide whether a
reassociated reduction stays within the reference's error.
"""
from __future__ import annotations
from collections.abc import Callable, Sequence
from dataclasses import dataclass
import torch
Impl = Callable[..., torch.Tensor]
@dataclass(frozen=True)
class ErrorScore:
"""One implementation's error against the fp64 evaluation of its math."""
rms: float
max_abs: float
mean: float
"""Signed mean error; a value far from 0 means the rounding is biased."""
def __str__(self) -> str:
return f"rms={self.rms:.3e} max={self.max_abs:.3e} mean={self.mean:+.3e}"
def score_against(reference_fp64: torch.Tensor, actual: torch.Tensor) -> ErrorScore:
"""Score ``actual`` against an fp64 evaluation of the same math."""
if reference_fp64.shape != actual.shape:
raise ValueError(
f"shape mismatch: reference {tuple(reference_fp64.shape)} vs "
f"actual {tuple(actual.shape)}"
)
error = actual.to(torch.float64) - reference_fp64
return ErrorScore(
rms=float(error.pow(2).mean().sqrt()),
max_abs=float(error.abs().max()),
mean=float(error.mean()),
)
def assert_error_no_worse_than_reference(
*,
reference_fp64: torch.Tensor,
baseline: torch.Tensor,
candidate: torch.Tensor,
max_error_ratio: float = 1.5,
max_bias_in_rms: float = 0.25,
label: str = "candidate",
) -> tuple[ErrorScore, ErrorScore]:
"""Admit ``candidate`` to the lossless tier on operator-level error.
``baseline`` is what the model runs today and ``reference_fp64`` the same
math evaluated in fp64. The candidate passes when it is no more than
``max_error_ratio`` times the baseline's RMS and max error -- the baseline
sets the scale, because the reference path is itself only one sample of
the rounding -- and when its mean error stays within ``max_bias_in_rms``
of its own RMS, which catches a rounding that always leans one way and
would accumulate over layers and steps.
"""
base = score_against(reference_fp64, baseline)
cand = score_against(reference_fp64, candidate)
for metric, base_value, cand_value in (
("RMS", base.rms, cand.rms),
("max", base.max_abs, cand.max_abs),
):
# An exactly-zero baseline error means the reference math is
# representable in the output dtype; hold the candidate to the same.
budget = base_value * max_error_ratio
if cand_value > budget:
raise AssertionError(
f"{label} is not lossless-tier: {metric} error {cand_value:.3e} "
f"exceeds {max_error_ratio}x the reference path's "
f"{base_value:.3e}. reference={base} candidate={cand}"
)
if cand.rms > 0 and abs(cand.mean) > max_bias_in_rms * cand.rms:
raise AssertionError(
f"{label} has a biased rounding: mean error {cand.mean:+.3e} is "
f"more than {max_bias_in_rms} of its RMS {cand.rms:.3e}, so the "
f"error accumulates instead of cancelling. candidate={cand}"
)
return base, cand
def assert_special_values_match(
*,
baseline: Impl,
candidate: Impl,
cases: dict[str, Sequence[torch.Tensor]],
label: str = "candidate",
) -> None:
"""Both paths agree on NaN, Inf, empty and non-contiguous inputs.
Agreement is structural: the same NaN positions, the same infinities with
the same signs, finite where the reference is finite, or the same
exception type. A fast path that swallows a NaN (``fmaxf``), returns zeros
where the reference returns NaN, or lets a NaN leak into rows it should
not reach is a defect rather than a tier.
A kernel that ignores a stride or aliases its inputs reads the wrong
elements, which is a magnitude error rather than a structural one: pass a
non-contiguous input to :func:`assert_error_no_worse_than_reference` as
well, where it shows up as an error far outside the budget.
It deliberately does not compare the magnitude of the finite values. A
lossless-tier path differs from the reference there by construction, and
judging that difference is :func:`assert_error_no_worse_than_reference`'s
job, on real activations rather than on these edge-case inputs. An earlier
version compared them with ``torch.allclose`` defaults, which rejected
every bf16 fast path this gate exists to admit.
"""
for name, inputs in cases.items():
try:
expected = baseline(*inputs)
except Exception as exc: # noqa: BLE001 - the reference's own behavior
expected_error: type[BaseException] | None = type(exc)
expected = None
else:
expected_error = None
try:
actual = candidate(*inputs)
except Exception as exc: # noqa: BLE001 - compared against the reference
if expected_error is None:
raise AssertionError(
f"{label} raised {type(exc).__name__} on {name!r} where "
f"the reference path returned a tensor"
) from exc
if type(exc) is not expected_error:
raise AssertionError(
f"{label} raised {type(exc).__name__} on {name!r}, the "
f"reference path raised {expected_error.__name__}"
) from exc
continue
if expected_error is not None:
raise AssertionError(
f"{label} returned a tensor on {name!r} where the reference "
f"path raised {expected_error.__name__}"
)
if actual.shape != expected.shape:
raise AssertionError(
f"{label} returns {tuple(actual.shape)} on {name!r} where the "
f"reference path returns {tuple(expected.shape)}"
)
if not torch.equal(actual.isnan(), expected.isnan()):
raise AssertionError(f"{label} moves NaNs on {name!r}")
# Signed, so an inf that flips direction is caught too.
for sign, what in ((1, "+Inf"), (-1, "-Inf")):
if not torch.equal(
actual == sign * torch.inf, expected == sign * torch.inf
):
raise AssertionError(f"{label} moves {what}s on {name!r}")
def discrete_flip_rate(baseline: torch.Tensor, candidate: torch.Tensor) -> float:
"""Fraction of entries whose discrete downstream result differs.
Pass the discrete values -- quantized codes, top-k indices, threshold
decisions -- not the continuous tensors they came from. One ulp upstream of
a quantizer or a ``skip-softmax`` threshold is no longer a rounding
difference, so a lossless-tier claim in front of one is judged on this rate.
"""
if baseline.shape != candidate.shape:
raise ValueError(
f"shape mismatch: {tuple(baseline.shape)} vs {tuple(candidate.shape)}"
)
if baseline.numel() == 0:
return 0.0
return float((baseline != candidate).to(torch.float64).mean())
@@ -1203,6 +1203,8 @@ TWO_GPU_CASES = [
ulysses_degree=1,
ring_degree=2,
),
# Keeps the pre-rename spelling of "lossless" so the compatibility
# alias is covered end to end; the case id is also a perf-baseline key.
replace(T2I_sampling_params, extras={"quality": "extra-high"}),
run_component_accuracy_check=False,
run_models_api_check=False,
@@ -361,6 +361,27 @@ class DiffusionTestCase:
):
raise ValueError(f"{self.id}: request warmup requires non-realtime metrics")
# A consistency golden records one path, and these were recorded on the
# reference one, so the case asks for it by name instead of inheriting
# whichever level is the server default. Without this the check would
# answer two questions at once -- "did the code regress" and "how far
# is the default level from the goldens" -- and spend its whole budget
# on the second: the default's own drift already sits at SSIM 0.91
# against goldens whose threshold is 0.92. A case that names a level
# keeps it; refreshing the goldens onto the default level is what
# removes the pin.
# Replaces rather than mutates: several cases share one module-level
# sampling-params instance.
if self.run_consistency_check and "quality" not in self.sampling_params.extras:
object.__setattr__(
self,
"sampling_params",
replace(
self.sampling_params,
extras={**self.sampling_params.extras, "quality": "exact"},
),
)
has_startup_lora = self.server_args.lora_path is not None
has_dynamic_lora = self.server_args.dynamic_lora_path is not None
has_second_lora = self.server_args.second_lora_path is not None
@@ -68,55 +68,127 @@ class TestQualityFusionBCGCompatibility(unittest.TestCase):
self.stage.server_args = SimpleNamespace(enable_breakable_cuda_graph=True)
self.stage.transformer = OtherTransformer2DModel()
self.stage.transformer_2 = None
self.stage._quality_fusions_mounted = False
self.stage._mounted_quality = "exact"
@staticmethod
def _batch(quality: str):
return SimpleNamespace(sampling_params=SimpleNamespace(quality=quality))
def test_rejects_fusion_levels_when_they_would_replace_captured_graph(self):
for quality in ("extra-high", "high"):
with self.subTest(quality=quality):
unmounted = []
handlers = (
(
"test fusion",
lambda _: True,
lambda transformer: unmounted.append(transformer),
),
)
def test_each_quality_level_gets_its_own_bcg_runner(self):
# A captured graph bakes in the fusions mounted when it was recorded,
# and the capture signature only covers the tensors. Warmup captures
# at the server's default level, so a request at another level must
# not reach those graphs: it gets an empty runner, and the eager
# fallback runs the fusion set it actually asked for.
self.stage._bcg_runners = {}
model = self.stage.transformer
built = []
with patch.object(
denoising_module, "_QUALITY_FUSION_HANDLERS", handlers
):
with self.assertRaisesRegex(ValueError, "lossless warmup graphs"):
self.stage._maybe_toggle_quality_fusions(self._batch(quality))
class _FakeRunner:
def __init__(self, transformer, device):
built.append(transformer)
self.assertEqual(unmounted, [self.stage.transformer])
self.assertFalse(self.stage._quality_fusions_mounted)
with (
patch.object(
denoising_module, "get_local_torch_device", return_value="cpu"
),
patch(
"sglang.multimodal_gen.runtime.breakable_cuda_graph.runner."
"DiffusionBreakableCudaGraphRunner",
_FakeRunner,
),
):
self.stage._mounted_quality = "lossless"
default_runner = self.stage._maybe_get_bcg_runner(model)
self.assertIs(self.stage._maybe_get_bcg_runner(model), default_runner)
def test_allows_high_when_model_has_no_dit_quality_fusions(self):
handlers = (("test fusion", lambda _: False, lambda _: None),)
self.stage._mounted_quality = "exact"
exact_runner = self.stage._maybe_get_bcg_runner(model)
with patch.object(denoising_module, "_QUALITY_FUSION_HANDLERS", handlers):
self.stage._maybe_toggle_quality_fusions(self._batch("high"))
self.assertIsNot(exact_runner, default_runner)
self.assertEqual(len(built), 2)
self.assertTrue(self.stage._quality_fusions_mounted)
def test_high_keeps_extra_high_fusions_mounted(self):
self.stage.server_args.enable_breakable_cuda_graph = False
def test_mounting_a_fusion_under_bcg_is_allowed(self):
# This used to raise: with one runner per module, switching level
# would have replayed the warmup level's kernels under another
# level's name. The runner key carries the level now, so the request
# simply runs.
mounted = []
handlers = (
("test fusion", lambda _: mounted.append(True) or True, lambda _: None),
(
"lossless",
"test fusion",
lambda t: mounted.append(t) or True,
lambda _: None,
),
)
with patch.object(denoising_module, "_QUALITY_FUSION_HANDLERS", handlers):
self.stage._maybe_toggle_quality_fusions(self._batch("extra-high"))
self.assertTrue(self.stage._quality_fusions_mounted)
self.stage._maybe_toggle_quality_fusions(self._batch("lossless"))
self.assertEqual(mounted, [self.stage.transformer])
self.assertEqual(self.stage._mounted_quality, "lossless")
def test_allows_high_when_model_has_no_dit_quality_fusions(self):
handlers = (("lossless", "test fusion", lambda _: False, lambda _: None),)
with patch.object(denoising_module, "_QUALITY_FUSION_HANDLERS", handlers):
self.stage._maybe_toggle_quality_fusions(self._batch("high"))
self.assertEqual(mounted, [True])
self.assertTrue(self.stage._quality_fusions_mounted)
self.assertEqual(self.stage._mounted_quality, "high")
def test_an_exact_request_unmounts_what_a_faster_tier_left_behind(self):
self.stage.server_args.enable_breakable_cuda_graph = False
mounted, unmounted = [], []
handlers = (
(
"lossless",
"test fusion",
lambda t: mounted.append(t) or True,
lambda t: unmounted.append(t),
),
)
with patch.object(denoising_module, "_QUALITY_FUSION_HANDLERS", handlers):
self.stage._maybe_toggle_quality_fusions(self._batch("lossless"))
self.assertEqual(mounted, [self.stage.transformer])
# Back to the default at the next batch boundary: the fusion has
# to come off, or an exact request silently keeps running it.
self.stage._maybe_toggle_quality_fusions(self._batch("exact"))
self.assertEqual(unmounted, [self.stage.transformer])
self.assertEqual(self.stage._mounted_quality, "exact")
def test_a_fusion_runs_only_from_the_tier_that_declares_it(self):
self.stage.server_args.enable_breakable_cuda_graph = False
mounted = []
handlers = (
(
"lossless",
"equivalent fusion",
lambda _: mounted.append("equivalent") or True,
lambda _: None,
),
(
"high",
"approximate fusion",
lambda _: mounted.append("approximate") or True,
lambda _: None,
),
)
with patch.object(denoising_module, "_QUALITY_FUSION_HANDLERS", handlers):
self.stage._maybe_toggle_quality_fusions(self._batch("lossless"))
self.assertEqual(mounted, ["equivalent"])
self.assertEqual(self.stage._mounted_quality, "lossless")
mounted.clear()
self.stage._maybe_toggle_quality_fusions(self._batch("high"))
# "high" is cumulative: it keeps the equivalent fusion and adds its own
self.assertEqual(mounted, ["equivalent", "approximate"])
self.assertEqual(self.stage._mounted_quality, "high")
def _fake_cache_dit_batch(*, is_warmup: bool) -> SimpleNamespace:
@@ -645,6 +717,8 @@ class TestDiffusionBCGPadding(unittest.TestCase):
def test_bcg_runner_cache_is_per_model_module(self):
self.stage.server_args = SimpleNamespace(enable_breakable_cuda_graph=True)
self.stage._bcg_runners = {}
# The cache key is (module, quality level); this one varies the module.
self.stage._mounted_quality = "lossless"
def fake_runner(model, device):
return SimpleNamespace(model=model, device=device)
@@ -0,0 +1,84 @@
# SPDX-License-Identifier: Apache-2.0
"""A mounted fast path falls back on inputs it cannot serve.
The fused RMSNorm+modulate launcher raises on anything outside its contract,
which is right for a caller that has already committed. The LTX-2 site has
not committed: it is request-gated, so an input it cannot serve has to go
down the reference chain instead of failing the request.
It did fail one. The site guarded on dtype and width only, while the launcher
also requires `scale` and `shift` to be contiguous `[B, 1, D]` rows, and
LTX-2.5's are not -- so every denoising step raised
"scale and shift must be contiguous [B, 1, D] tensors" as soon as the fusion
was mounted. Nothing caught it earlier because the fusion only mounted above
the default level.
"""
import sys
import unittest
import pytest
import torch
from sglang.kernels.ops.diffusion import can_use_fused_rmsnorm_modulation
@unittest.skipUnless(torch.cuda.is_available(), "needs CUDA")
class TestFusedRmsNormModulationGuard(unittest.TestCase):
"""The guard answers exactly the question the launcher would raise on."""
def setUp(self):
torch.manual_seed(0)
self.x = torch.randn(2, 16, 2048, device="cuda", dtype=torch.bfloat16)
self.rows = torch.randn(2, 1, 2048, device="cuda", dtype=torch.bfloat16)
def test_admits_the_layout_the_kernel_serves(self):
self.assertTrue(
can_use_fused_rmsnorm_modulation(self.x, self.rows, self.rows.clone())
)
def test_rejects_modulation_rows_that_are_not_contiguous(self):
# LTX-2.5's shape: the rows arrive as a stride into a wider tensor.
wide = torch.randn(2, 1, 4096, device="cuda", dtype=torch.bfloat16)
strided = wide[:, :, ::2]
self.assertFalse(strided.is_contiguous())
self.assertFalse(can_use_fused_rmsnorm_modulation(self.x, strided, self.rows))
def test_rejects_a_per_token_modulation(self):
per_token = torch.randn(2, 16, 2048, device="cuda", dtype=torch.bfloat16)
self.assertFalse(can_use_fused_rmsnorm_modulation(self.x, per_token, self.rows))
def test_rejects_mismatched_dtype_and_width(self):
self.assertFalse(
can_use_fused_rmsnorm_modulation(self.x, self.rows.float(), self.rows)
)
narrow = torch.randn(2, 16, 1024, device="cuda", dtype=torch.bfloat16)
rows = torch.randn(2, 1, 1024, device="cuda", dtype=torch.bfloat16)
self.assertFalse(can_use_fused_rmsnorm_modulation(narrow, rows, rows))
def test_whatever_the_guard_rejects_the_launcher_would_have_raised(self):
# The guard is only worth having if it is exactly the launcher's
# contract: a rejected input must be one the launcher refuses, and an
# admitted one must go through.
from sglang.kernels.ops.diffusion import fused_rmsnorm_scale_shift_bitexact
weight = torch.ones(2048, device="cuda", dtype=torch.bfloat16)
wide = torch.randn(2, 1, 4096, device="cuda", dtype=torch.bfloat16)
for name, scale, shift in (
("non-contiguous rows", wide[:, :, ::2], self.rows),
("per-token modulation", self.x, self.rows),
("fp32 rows", self.rows.float(), self.rows),
):
with self.subTest(name):
self.assertFalse(can_use_fused_rmsnorm_modulation(self.x, scale, shift))
with self.assertRaises(RuntimeError):
fused_rmsnorm_scale_shift_bitexact(
self.x, weight, scale, shift, 1e-6
)
self.assertTrue(can_use_fused_rmsnorm_modulation(self.x, self.rows, self.rows))
fused_rmsnorm_scale_shift_bitexact(self.x, weight, self.rows, self.rows, 1e-6)
if __name__ == "__main__":
sys.exit(pytest.main([__file__, "-v"]))
@@ -248,7 +248,7 @@ def test_hybrid_override_loads_ref_config_and_admits_all_native_tasks(weights):
stage = MiniMaxH3PartitionAdmissionStage(pipeline.release_metadata)
for task in ("t2va", "fl2va", "ref2va"):
batch = SimpleNamespace(
sampling_params=SimpleNamespace(task=task, quality="lossless"),
sampling_params=SimpleNamespace(task=task, quality="exact"),
num_inference_steps=50,
)
assert (
@@ -440,7 +440,7 @@ def test_admission_rejects_steps_exceeding_online_adaln_gpu_plans():
server_args = _quality_server_args()
server_args.minimax_h3_adaln_online = True
batch = SimpleNamespace(
sampling_params=SimpleNamespace(task="t2va", quality="lossless"),
sampling_params=SimpleNamespace(task="t2va", quality="exact"),
num_inference_steps=50,
is_warmup=False,
)
@@ -454,15 +454,15 @@ def test_admission_rejects_steps_exceeding_online_adaln_gpu_plans():
assert stage.forward(batch, server_args) is batch
def test_extra_high_quality_does_not_enable_h3_cache_dit():
def test_lossless_quality_does_not_enable_h3_cache_dit():
stage = MiniMaxH3DenoisingStage.__new__(MiniMaxH3DenoisingStage)
stage.server_args = SimpleNamespace(enable_breakable_cuda_graph=False)
stage._cache_dit_enabled = False
stage._minimax_h3_cache_mode = None
stage._minimax_h3_quality = "lossless"
stage._minimax_h3_quality = "exact"
batch = SimpleNamespace(
sampling_params=SimpleNamespace(
quality="extra-high",
quality="lossless",
_explicit_fields={"quality"},
enable_cache_dit=None,
cache_dit_params=None,
@@ -475,7 +475,7 @@ def test_extra_high_quality_does_not_enable_h3_cache_dit():
with patch.object(DenoisingStage, "_cache_dit_requested", return_value=True):
stage._maybe_enable_cache_dit(50, batch)
assert stage._minimax_h3_quality == "extra-high"
assert stage._minimax_h3_quality == "lossless"
assert stage._minimax_h3_cache_mode is None
assert not stage._cache_dit_enabled
@@ -534,7 +534,7 @@ def test_quality_admission_fails_closed_outside_validated_request():
server_args.attention_backend = "sage_attn"
assert stage.forward(batch, server_args) is batch
batch.sampling_params.quality = "extra-high"
batch.sampling_params.quality = "lossless"
assert stage.forward(batch, server_args) is batch
batch.sampling_params.quality = "ultra"
@@ -0,0 +1,60 @@
# SPDX-License-Identifier: Apache-2.0
"""The worker states its approximate-numerics settings instead of inheriting them.
TF32 convolutions and bf16 reduced-precision reduction change what is computed,
so leaving them at whatever the installed PyTorch defaults to makes the exact
tier's contract depend on the build. The policy sets both explicitly.
"""
import sys
import unittest
import pytest
import torch
from sglang.multimodal_gen.runtime.utils.numerics_policy import apply_numerics_policy
class TestNumericsPolicy(unittest.TestCase):
def setUp(self):
self._cudnn_tf32 = torch.backends.cudnn.allow_tf32
self._bf16_reduction = (
torch.backends.cuda.matmul.allow_bf16_reduced_precision_reduction
)
def tearDown(self):
torch.backends.cudnn.allow_tf32 = self._cudnn_tf32
torch.backends.cuda.matmul.allow_bf16_reduced_precision_reduction = (
self._bf16_reduction
)
def test_both_switches_follow_the_policy(self):
for tf32, bf16 in ((False, False), (True, True), (False, True)):
with self.subTest(tf32=tf32, bf16=bf16):
apply_numerics_policy(
allow_cudnn_tf32=tf32,
allow_bf16_reduced_precision_reduction=bf16,
)
self.assertEqual(torch.backends.cudnn.allow_tf32, tf32)
self.assertEqual(
torch.backends.cuda.matmul.allow_bf16_reduced_precision_reduction,
bf16,
)
def test_the_strict_setting_is_reachable_from_any_starting_state(self):
torch.backends.cudnn.allow_tf32 = True
torch.backends.cuda.matmul.allow_bf16_reduced_precision_reduction = True
apply_numerics_policy(
allow_cudnn_tf32=False,
allow_bf16_reduced_precision_reduction=False,
)
self.assertFalse(torch.backends.cudnn.allow_tf32)
self.assertFalse(
torch.backends.cuda.matmul.allow_bf16_reduced_precision_reduction
)
if __name__ == "__main__":
sys.exit(pytest.main([__file__, "-v"]))
@@ -64,7 +64,7 @@ def test_runtime_sampling_quality_preserves_the_openai_default():
assert _runtime_sampling_quality(None) is None
assert _runtime_sampling_quality("auto") is None
assert _runtime_sampling_quality("lossless") == "lossless"
assert _runtime_sampling_quality("extra-high") == "extra-high"
assert _runtime_sampling_quality("lossless") == "lossless"
assert _runtime_sampling_quality("high") == "high"
@@ -94,14 +94,14 @@ class TestDiffusionPrecisionConsistency(unittest.TestCase):
self.assertEqual(
resolve_decode_precision(
self._server_args(vae_decode_precision_high="bf16"),
quality="extra-high",
quality="lossless",
),
torch.float16,
)
self.assertEqual(
resolve_decode_precision(
self._server_args(vae_decode_precision_high="bf16"),
quality="lossless",
quality="exact",
),
torch.float16,
)
@@ -0,0 +1,168 @@
# SPDX-License-Identifier: Apache-2.0
"""The lossless-tier admission template admits re-association, not precision loss.
Each fusion used to carry its own hand-picked atol/rtol, which cannot tell a
reassociated reduction from a kernel that quietly dropped a few mantissa bits:
both land "close". The template scores every implementation against the same
fp64 evaluation and judges the candidate relative to the path the model
already runs, so the question it answers is the tier's own question.
"""
import sys
import unittest
import pytest
import torch
from sglang.multimodal_gen.test.quality_tier_admission import (
assert_error_no_worse_than_reference,
assert_special_values_match,
discrete_flip_rate,
score_against,
)
def _row_sums(x: torch.Tensor, dtype: torch.dtype) -> torch.Tensor:
"""Sequential row sum in ``dtype`` -- the association under test."""
return x.to(dtype).sum(dim=-1)
class TestOperatorLevelGate(unittest.TestCase):
def setUp(self):
torch.manual_seed(0)
# Outlier channels are the point: DiT activations have them, and they
# decide whether a reassociated reduction stays within the reference.
self.x = torch.randn(64, 512, dtype=torch.float32)
self.x[:, ::97] *= 300.0
self.reference_fp64 = self.x.to(torch.float64).sum(dim=-1)
def test_admits_a_reassociated_reduction(self):
baseline = _row_sums(self.x, torch.float32)
# Pairwise summation: same operands and accumulation precision, a
# different order -- the shape of a lossless-tier fusion.
pairwise = self.x.to(torch.float32).reshape(64, -1, 2).sum(dim=-1).sum(dim=-1)
base, cand = assert_error_no_worse_than_reference(
reference_fp64=self.reference_fp64,
baseline=baseline,
candidate=pairwise,
)
self.assertGreater(base.rms, 0.0)
self.assertLess(cand.rms, base.rms * 1.5)
def test_rejects_a_reduction_that_accumulates_in_bf16(self):
baseline = _row_sums(self.x, torch.float32)
degraded = _row_sums(self.x, torch.bfloat16).to(torch.float32)
with self.assertRaisesRegex(AssertionError, "not lossless-tier"):
assert_error_no_worse_than_reference(
reference_fp64=self.reference_fp64,
baseline=baseline,
candidate=degraded,
)
def test_rejects_a_rounding_that_always_leans_one_way(self):
# A one-sided error small enough to stay inside the magnitude budget:
# the shape of truncation instead of round-to-nearest-even, or of an
# approximate transcendental whose error keeps its sign. It cannot be
# caught by a tolerance, because per-element it is tiny -- but it
# accumulates over layers and steps where a symmetric error cancels.
baseline = _row_sums(self.x, torch.float32)
leaning = baseline + 5e-5
base = score_against(self.reference_fp64, baseline)
cand = score_against(self.reference_fp64, leaning)
self.assertLess(cand.rms, base.rms * 1.5, "stays inside the magnitude budget")
with self.assertRaisesRegex(AssertionError, "biased rounding"):
assert_error_no_worse_than_reference(
reference_fp64=self.reference_fp64,
baseline=baseline,
candidate=leaning,
)
def test_score_reports_signed_mean_separately_from_magnitude(self):
score = score_against(
torch.zeros(4, dtype=torch.float64), torch.full((4,), 0.5)
)
self.assertAlmostEqual(score.mean, 0.5)
self.assertAlmostEqual(score.rms, 0.5)
self.assertAlmostEqual(score.max_abs, 0.5)
class TestSpecialValues(unittest.TestCase):
def test_a_fast_path_that_swallows_nan_is_rejected(self):
cases = {"nan row": (torch.tensor([1.0, float("nan"), 3.0]),)}
assert_special_values_match(
baseline=torch.relu, candidate=torch.relu, cases=cases
)
with self.assertRaisesRegex(AssertionError, "moves NaNs"):
assert_special_values_match(
baseline=torch.relu,
candidate=lambda x: torch.nan_to_num(torch.relu(x)),
cases=cases,
)
def test_a_fast_path_that_ignores_strides_is_rejected(self):
# Reading the wrong elements is a magnitude error, not a structural
# one, so it is the operator-level gate that has to catch it.
torch.manual_seed(0)
strided = torch.randn(8, 16)[:, ::2]
reference = (strided.double() * 2).contiguous()
baseline = strided * 2
# Read the same storage as if it were contiguous -- what a kernel that
# takes a data pointer and assumes packed rows actually does. (A
# `.flatten()` would quietly copy and get the right answer.)
ignores_stride = (
torch.as_strided(strided, strided.shape, (strided.shape[1], 1)) * 2
)
with self.assertRaisesRegex(AssertionError, "not lossless-tier"):
assert_error_no_worse_than_reference(
reference_fp64=reference,
baseline=baseline,
candidate=ignores_stride,
label="stride-ignoring kernel",
)
def test_a_rounding_level_difference_is_not_a_special_value_failure(self):
# The gate asks whether the special values behave the same, not
# whether the numbers match: a lossless-tier path differs there by
# construction, and judging that is the operator gate's job.
torch.manual_seed(0)
x = torch.randn(32, 8)
cases = {"plain": (x,), "with a NaN": (torch.cat([x[:1] * float("nan"), x]),)}
assert_special_values_match(
baseline=lambda t: t * 3.0,
candidate=lambda t: (t.to(torch.bfloat16) * 3.0).to(t.dtype),
cases=cases,
)
def test_raising_where_the_reference_returns_is_rejected(self):
cases = {"empty": (torch.zeros(0),)}
def explodes(x):
raise RuntimeError("no empty support")
with self.assertRaisesRegex(AssertionError, "RuntimeError"):
assert_special_values_match(
baseline=lambda x: x, candidate=explodes, cases=cases
)
class TestDiscreteDownstream(unittest.TestCase):
def test_flip_rate_counts_decisions_not_distance(self):
baseline = torch.tensor([0, 1, 2, 3])
self.assertEqual(discrete_flip_rate(baseline, baseline.clone()), 0.0)
self.assertEqual(discrete_flip_rate(baseline, torch.tensor([0, 1, 2, 9])), 0.25)
def test_empty_input_has_no_flips(self):
empty = torch.zeros(0, dtype=torch.long)
self.assertEqual(discrete_flip_rate(empty, empty), 0.0)
if __name__ == "__main__":
sys.exit(pytest.main([__file__, "-v"]))
@@ -0,0 +1,138 @@
# SPDX-License-Identifier: Apache-2.0
"""The tier boundary holds for the shapes the request-gated fusions actually use.
`_QUALITY_FUSION_HANDLERS` assigns each fusion a tier from what it does to the
numbers. This pins that reading to measurement: the transformations the
lossless-tier fusions perform pass the admission gate, and the ones this
repository classifies as approximate fail it, on activations shaped like a
DiT's -- outlier channels included, because they are what decides whether a
reassociated reduction stays within the reference's error.
Measured on real FLUX.1-dev MLP activations (4096x3072 bf16, max/p99 = 8.1)
the two groups separate by three to six orders of magnitude, so these verdicts
do not sit near the 1.5x threshold.
"""
import sys
import unittest
import pytest
import torch
from sglang.multimodal_gen.test.quality_tier_admission import (
assert_error_no_worse_than_reference,
)
def _dit_activations(rows: int, cols: int) -> torch.Tensor:
"""BF16 activations with the outlier channels a DiT block carries."""
generator = torch.Generator(device="cuda").manual_seed(0)
x = torch.randn(rows, cols, device="cuda", dtype=torch.float32, generator=generator)
x[:, ::97] *= 8.0
return x.to(torch.bfloat16)
@unittest.skipUnless(torch.cuda.is_available(), "needs CUDA")
class TestLosslessTierTransformations(unittest.TestCase):
"""What the lossless-tier fusions do stays inside the gate."""
def test_a_gemm_epilogue_fusion_is_admitted(self):
# fused linear+GELU: the cublasLt epilogue rounds once where the
# reference rounds the GEMM output and then the GELU output.
x = _dit_activations(1024, 1024)
weight = _dit_activations(2048, 1024)
bias = _dit_activations(1, 2048)[0]
reference = torch.nn.functional.gelu(
x.double() @ weight.double().t() + bias.double(), approximate="tanh"
)
baseline = torch.nn.functional.gelu(
torch.nn.functional.linear(x, weight, bias), approximate="tanh"
)
candidate = torch._addmm_activation(bias, x, weight.t(), use_gelu=True)
assert_error_no_worse_than_reference(
reference_fp64=reference,
baseline=baseline,
candidate=candidate,
label="fused linear+GELU",
)
def test_a_reassociated_reduction_is_admitted(self):
x = _dit_activations(512, 2048).float()
assert_error_no_worse_than_reference(
reference_fp64=x.double().sum(dim=-1),
baseline=x.sum(dim=-1),
candidate=x.reshape(x.shape[0], -1, 2).sum(-1).sum(-1),
label="reassociated fp32 reduction",
)
def test_the_gated_residual_rounding_is_admitted(self):
# Helios / LingBot: the reference multiplies the FP32 gate and update
# and rounds once at the end; the fused kernel needs one dtype, so it
# rounds both to BF16 first. One rounding moves earlier -- which the
# taxonomy can only call equivalent if it measures that way.
residual = _dit_activations(1024, 1024)
update = _dit_activations(1024, 1024).float()
gate = torch.rand(1024, 1, device="cuda") * 0.2
assert_error_no_worse_than_reference(
reference_fp64=residual.double() + update.double() * gate.double(),
baseline=residual + (update * gate).to(residual.dtype),
candidate=residual + (update.to(residual.dtype) * gate.to(residual.dtype)),
label="per-token gated residual",
)
@unittest.skipUnless(torch.cuda.is_available(), "needs CUDA")
class TestApproximateTransformationsAreRejected(unittest.TestCase):
"""What the high-tier fusions do cannot pass as a rounding difference."""
def _assert_rejected(self, label, reference_fp64, baseline, candidate):
with self.assertRaisesRegex(AssertionError, "not lossless-tier"):
assert_error_no_worse_than_reference(
reference_fp64=reference_fp64,
baseline=baseline,
candidate=candidate,
label=label,
)
def test_norm_statistics_rounded_through_bf16(self):
# Ideogram's BF16-native gate RMSNorm against F.rms_norm's fp32 stats.
x = _dit_activations(512, 2048)
self._assert_rejected(
"bf16 norm statistics",
x.double().pow(2).mean(dim=-1),
x.float().pow(2).mean(dim=-1),
x.pow(2).mean(dim=-1).float(),
)
def test_a_gemm_fed_bf16_where_the_reference_promotes_to_fp32(self):
# SANA-Video's BF16-input linear attention.
a = _dit_activations(512, 1024)
b = _dit_activations(512, 1024).t().contiguous()
self._assert_rejected(
"BF16-input attention GEMM",
a.double() @ b.double(),
a.float() @ b.float(),
(a @ b).float(),
)
def test_quantizing_before_the_reference_intermediate_exists(self):
# FLUX.2's NVFP4 FC1+SwiGLU+quant.
x = _dit_activations(512, 2048)
scale = x.abs().amax(dim=-1, keepdim=True).float() / 448.0
quantized = (x.float() / scale).clamp(-448, 448).to(
torch.float8_e4m3fn
).float() * scale
self._assert_rejected(
"quantized FC2 input",
torch.nn.functional.silu(x.double()),
torch.nn.functional.silu(x.float()),
torch.nn.functional.silu(quantized),
)
if __name__ == "__main__":
sys.exit(pytest.main([__file__, "-v"]))
@@ -30,7 +30,8 @@ from sglang.multimodal_gen.configs.sample.sampling_params import (
SamplingParams,
SkipSoftmaxParams,
_json_safe,
quality_allows_kernel_fusions,
normalize_quality,
quality_allows,
resolve_skip_softmax_params,
)
from sglang.multimodal_gen.configs.sample.spectrum import SpectrumParams
@@ -54,17 +55,30 @@ class TestSamplingParamsValidate(unittest.TestCase):
SamplingParams(num_outputs_per_prompt=0)
def test_quality_defaults_to_lossless(self):
# The default runs every fast path that keeps the reference math;
# "exact" is the opt-in for bit-identical output.
self.assertEqual(SamplingParams().quality, "lossless")
def test_quality_levels_are_cumulative(self):
self.assertEqual(QUALITY_LEVELS, ("lossless", "extra-high", "high"))
self.assertEqual(QUALITY_LEVELS, ("exact", "lossless", "high"))
for quality in QUALITY_LEVELS:
with self.subTest(quality=quality):
self.assertEqual(SamplingParams(quality=quality).quality, quality)
self.assertFalse(quality_allows_kernel_fusions("lossless"))
self.assertTrue(quality_allows_kernel_fusions("extra-high"))
self.assertTrue(quality_allows_kernel_fusions("high"))
# A request admits the fast paths of its own tier and of every
# stricter one, so the levels stay cumulative.
for request, admits in (
("exact", {"exact"}),
("lossless", {"exact", "lossless"}),
("high", {"exact", "lossless", "high"}),
):
for tier in QUALITY_LEVELS:
with self.subTest(request=request, tier=tier):
self.assertEqual(quality_allows(request, tier), tier in admits)
def test_extra_high_is_accepted_as_the_former_name_of_lossless(self):
self.assertEqual(SamplingParams(quality="extra-high").quality, "lossless")
self.assertEqual(normalize_quality("extra-high"), "lossless")
def test_quality_rejects_invalid_values(self):
for bad in ("ultra", "draft", "fast", "", True, 1):
@@ -371,7 +385,7 @@ class TestSamplingParamsCliArgs(unittest.TestCase):
def test_quality_is_request_scoped_cli_arg(self):
self.assertNotIn("quality", self._parse_cli_kwargs([]))
for quality in ("extra-high", "high"):
for quality in ("exact", "lossless", "high", "extra-high"):
with self.subTest(quality=quality):
self.assertEqual(
self._parse_cli_kwargs(["--quality", quality])["quality"], quality