diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 394459516f..f86911aef9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,5 +1,5 @@ default_stages: [pre-commit, pre-push, manual] -exclude: ^(python/sglang/multimodal_gen/csrc|python/sglang/kernels/ops/diffusion/render|python/sglang/kernels/ops/attention/flash_attn/cute) +exclude: ^(python/sglang/multimodal_gen/csrc|python/sglang/kernels/ops/diffusion/render|python/sglang/kernels/ops/attention/flash_attn/cute|python/sglang/kernels/jit/csrc/diffusion/vsa_block_sparse_sm100) repos: - repo: https://github.com/pre-commit/pre-commit-hooks diff --git a/docs/cookbook/diffusion/MiniMax/MiniMax-H3.mdx b/docs/cookbook/diffusion/MiniMax/MiniMax-H3.mdx index 46fac1627a..67032d58db 100644 --- a/docs/cookbook/diffusion/MiniMax/MiniMax-H3.mdx +++ b/docs/cookbook/diffusion/MiniMax/MiniMax-H3.mdx @@ -847,68 +847,71 @@ The nine sigma grid points give eight denoiser evaluations. These settings must match the fused heads; the server rejects a different step count or shift. Use `quality: "lossless"` in requests. -## 6. FastH3: 4-step distilled preview +## 6. FastH3: 8-step distilled -[FastVideo/FastVideo-FastH3-4-step-Preview-v1-VSA-DataFree](https://huggingface.co/FastVideo/FastVideo-FastH3-4-step-Preview-v1-VSA-DataFree) -is a 4-step DMD2 distillation of MiniMax-H3, trained data-free with Video -Sparse Attention (VSA) at 0.9 sparsity and 64-token tiles. Only the T2VA -capability was distilled: requests must use `task: "t2va"`, and `fl2va` / -`ref2va` requests are rejected. The checkpoint inherits the MiniMax-H3 -Community License. +[FastVideo/FastVideo-FastH3-8-Step-V2](https://huggingface.co/FastVideo/FastVideo-FastH3-8-Step-V2) +is an 8-step DMD2 distillation of MiniMax-H3, trained data-free with Video +Sparse Attention (VSA) at 0.8 sparsity and 64-token tiles, with video / audio +scheduler shifts 10 / 3. Only the T2VA capability was distilled: requests must +use `task: "t2va"`, and `fl2va` / `ref2va` requests are rejected. The +checkpoint inherits the MiniMax-H3 Community License. Pass the repository directly to `--model-path`. The flat native-Diffusers upload is materialized into the base-H3 layout through a registered model overlay; the only non-symlink step is a one-time re-serialization of the roughly 10 GB video VAE on first launch. -```bash 4×B300 VSA-H3 +```bash 4×GB300 VSA-H3 sglang serve \ - --model-path FastVideo/FastVideo-FastH3-4-step-Preview-v1-VSA-DataFree \ + --model-path FastVideo/FastVideo-FastH3-8-Step-V2 \ --num-gpus 4 \ - --attention-backend video_sparse_attn_h3 \ - --attention-backend-config '{"VSA_sparsity": 0.9}' \ + --component-attention-backends transformer=video_sparse_attn_h3 \ --port 30010 ``` Requests use the same asynchronous video endpoint as the base model, with `task: "t2va"`, `conditions: []`, and a target such as `{"short_edge": 768, "aspect_ratio": "16:9", "duration_seconds": 5.0}`. The -request default is `num_inference_steps: 5`: five points on the standard -shift-12/shift-3 sigma grid, i.e. the four distilled DiT evaluations. Any -other step count is rejected. +request default is `num_inference_steps: 9`: nine sigma grid points, i.e. the +eight distilled DiT evaluations on the checkpoint's trained rungs +`[999, 874, 749, 624, 500, 375, 250, 125]` (unshifted timesteps over 1000, +shifted once per modality, as FastVideo runs them). Any other step count is +rejected. The trained sparsity 0.8 is the default; `--attention-backend-config +'{"VSA_sparsity": ...}'` overrides it. FastH3 requests default to +`quality: "lossless"`, which fuses the VAE decoder's per-head QK RMSNorm and +RoPE and runs its attention on cuDNN SDPA; `quality: "exact"` keeps the +reference decoder kernels. At every quality level FastH3 decodes each GPU's +spatial VAE tiles as one batch rather than one at a time, which changes GEMM +shapes and so the output at rounding level; base MiniMax-H3 decodes tile by +tile. -`video_sparse_attn_h3` (VSA-H3) is the trained sparse policy: an in-tree -Triton block-sparse kernel (SM90 / SM100 / SM103) over segment-pure prefix -tiles and (4, 4, 4) video tiles, driven by the checkpoint's trained -`to_gate_compress` compression branch. Only the DiT runs sparse; the token -refiner, text encoder, and VAEs keep their dense defaults. Ulysses sequence -parallelism is supported. See +`video_sparse_attn_h3` (VSA-H3) is the trained sparse policy: in-tree +block-sparse kernels (FastVideo's native tcgen05 forward on SM100 / SM103, a +Triton tile-64 kernel on SM90) over segment-pure prefix tiles and (4, 4, 4) +video tiles, driven by the checkpoint's trained +`to_gate_compress` compression branch. Select it for the transformer only, as +with cube sparse attention: the packed DiT blocks run sparse while the token +refiner falls back to dense FA inside the backend, and the text encoder and +VAEs keep their own backends. Ulysses sequence parallelism is supported. See [Attention Backends](/docs/sglang-diffusion/attention_backends) for `VSA_sparsity`, `vsa_mode`, `vsa_dense_first_n_steps`, and `vsa_dense_layers`. Every dense backend that runs on base H3 (`fa`, `torch_sdpa`, ...) also runs on the FastH3 weights without VSA flags, and `sglang generate` takes the same flags as `sglang serve`. -Measured latencies for the 4× B300 recipe are in -[FastH3 on B300](#fasth3-on-b300). +Measured latencies for the 4× GB300 recipe are in +[FastH3 on GB300](#fasth3-on-gb300). -FastH3 rejects deployment options that do not apply to the distilled preview -instead of silently ignoring them: `--model-variant`, `quality: "high"`, -`fl2va` / `ref2va` requests, and, with VSA-H3, `--ring-degree` greater than 1, -`torch.compile`, and breakable CUDA graph execution. - - -The sibling `FastVideo/FastVideo-FastH3-4-step-Preview-v1-LoRA` adapters carry -full-rank `.diff` / `.diff_b` deltas and `set_weight` gate tensors beyond the -LoRA contract; `--lora-path` rejects them with an explicit error. Serve the -merged VSA-DataFree checkpoint above instead. - +FastH3 rejects deployment options that do not apply to the distilled +checkpoint instead of silently ignoring them: `--model-variant`, +`quality: "high"`, `fl2va` / `ref2va` requests, and, with VSA-H3, +`--ring-degree` greater than 1, `torch.compile`, and breakable CUDA graph +execution. -Upstream labels this checkpoint a preview. Quality gaps versus base H3 on hard -motion and fine detail are properties of the released distillation, not of the -SGLang port. Use base MiniMax-H3 when output quality matters more than -latency. +Quality gaps versus base H3 on hard motion and fine detail are properties of +the released distillation, not of the SGLang port. Use base MiniMax-H3 when +output quality matters more than latency. ## 7. VDN-H3: hybrid attention, 8-step distill @@ -1008,18 +1011,20 @@ done `quality` is a cumulative request-scoped optimization parameter with three levels: -- `"exact"` (default): the exact reference path. Output is bit-exact +- `"exact"`: the exact reference path. Output is bit-exact against the reference implementation and the CI ground truth. -- `"lossless"`: includes the global fusion-only tier but does not enable - Cache-DiT or another approximate optimization. MiniMax-H3 currently has no - request-gated fusion site, so its denoise path is the same as `exact`. +- `"lossless"` (default): includes the global fusion-only tier but does not enable + Cache-DiT or another approximate optimization. The denoise path is the same + as `exact`; VAE decode fuses the ViT decoder's per-head QK RMSNorm and + RoPE into one launch and runs its attention on cuDNN SDPA (rounding-level + differences only). - `"high"`: the audited accelerated path. Quality is guaranteed (the audited Cache-DiT configuration measures SSIM 0.931 / PSNR 28.16 dB against - `lossless`), but output is no longer bit-identical to the reference. + `exact`), but output is no longer bit-identical to the reference. One resident server serves all three levels; a `quality: "high"` request mounts its audited Cache-DiT policy at the batch boundary, and a later -`quality: "lossless"` or `quality: "lossless"` request removes the hooks +`quality: "exact"` or `quality: "lossless"` request removes the hooks before denoising. Start the validated server once: @@ -1044,23 +1049,25 @@ Then choose a request level: - + -Native denoising with no feature-cache approximation. This is the default; -omitting the field is equivalent. +Native denoising and VAE decode with no feature-cache approximation or +quality-gated VAE fusions. ```json Request field { - "quality": "lossless" + "quality": "exact" } ``` - + The global fusion-only tier. It does not enable MiniMax-H3 Cache-DiT and -currently follows the same H3 denoise path as `exact`. +follows the same H3 denoise path as `exact`; only the VAE decoder's fused +QK RMSNorm + RoPE and cuDNN SDPA attention differ. +Omitting the field selects this level. ```json Request field { @@ -1073,7 +1080,7 @@ currently follows the same H3 denoise path as `exact`. The audited accelerated path. Use it when you can trade bit-exactness for -latency while keeping output closest to the same-seed lossless trajectory. +latency while keeping output closest to the same-seed exact trajectory. ```json Request field { @@ -1087,10 +1094,10 @@ latency while keeping output closest to the same-seed lossless trajectory. The measured trade-off is: -| `quality` | Mean
inference
latency | Speedup | SSIM vs
lossless | PSNR vs
lossless | Expected
trade-off | +| `quality` | Mean
inference
latency | Speedup | SSIM vs
exact | PSNR vs
exact | Expected
trade-off | | --- | ---: | ---: | ---: | ---: | --- | | `exact` | 75.10 s | 1.00× | 1.000 | exact | Native reference path | -| `lossless` | Not separately measured | — | Same H3 denoise path | Same H3 denoise path | Fusion-only tier; no H3-specific request-gated site yet | +| `lossless` | Not separately measured | — | Same H3 denoise path | Same H3 denoise path | Fusion-only tier; VAE decode fuses QK RMSNorm + RoPE and uses cuDNN SDPA (rounding-level differences) | | `high` | 53.70 s | 1.40× | 0.931 | 28.16 dB | Smallest same-seed visual change | These numbers use 1344×768, 124-frame, 24 fps T2VA with 50 inference steps, @@ -1608,29 +1615,32 @@ python3 -m sglang.multimodal_gen.benchmarks.bench_serving \ | Ref2VA | FP8 | fold | 112.0 s | 34.44 s | **27.12 s** | 52,816 MB | | Ref2VA | FP8 | replicate | 116.0 s | 33.42 s | **27.12 s** | 93,396 MB | -### FastH3 on B300 +### FastH3 on GB300 -The same 4× B300 host served [FastH3](#6-fasth3-4-step-distilled-preview) -with the VSA-H3 recipe above (1344×768 at 24 fps with audio, `task: "t2va"`, -`num_inference_steps: 5`, seed 1000, eager BF16, Ulysses4, `VSA_sparsity` 0.9). -E2E is the client wall clock of a `/v1/videos` request including decode, -muxing, and file output, median of three requests after one warm request; -the stage columns are the server timings of the same request. H3 aligns the -requested durations to 124, 243, and 362 frames. Client RTF is E2E divided by -the video duration: +A 4× GB300 host served [FastH3](#6-fasth3-8-step-distilled) with the VSA-H3 +recipe above (1344×768 at 24 fps with audio, `task: "t2va"`, +`num_inference_steps: 9`, the default `quality: "extra-high"` and sparsity 0.8, +eager BF16, Ulysses4, `NCCL_NVLS_ENABLE=0`), measured with the MP4 encoded +while the VAE decodes +([sgl-project/sglang#41819](https://github.com/sgl-project/sglang/pull/41819)). +The workload is 72 requests at +concurrency 1 after one warm request per shape: six scene families, 1,000- and +10,000-token prompts, 5 / 10 / 15 s, two seeds, every prompt unique so no +conditioning cache hits. E2E is the client wall clock from submission to the +last byte of the MP4; the stage columns are the server timings of the same +requests. Each cell is the median of the 12 requests of that shape: -| Requested / aligned | Encoder | Denoise (4 forwards) | Decode | Transport + MP4 | E2E | Client RTF | Peak/GPU | -| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | -| 5 s / 124 | 0.07 s | 2.18 s | 0.87 s | 0.9 s | **4.1 s** | 0.79 | 95,744 MB | -| 10 s / 243 | 0.07 s | 4.83 s | 1.71 s | 1.3 s | **8.0 s** | 0.79 | 102,666 MB | -| 15 s / 362 | 0.07 s | 8.80 s | 2.56 s | 1.8 s | **13.3 s** | 0.88 | 110,774 MB | +| Prompt / requested / aligned | Text encoder | Denoise (8 forwards) | Decode (video + audio) | E2E | Peak/GPU | +| --- | ---: | ---: | ---: | ---: | ---: | +| 1K / 5 s / 124 | 0.10 s | 3.84 s | 0.56 s | **5.20 s** | 94,468 MB | +| 1K / 10 s / 243 | 0.10 s | 8.45 s | 1.12 s | **10.74 s** | 97,628 MB | +| 1K / 15 s / 362 | 0.10 s | 14.85 s | 1.72 s | **18.24 s** | 100,246 MB | +| 10K / 5 s / 124 | 0.18 s | 6.31 s | 0.55 s | **7.90 s** | 94,510 MB | +| 10K / 10 s / 243 | 0.18 s | 12.35 s | 1.12 s | **14.98 s** | 98,128 MB | +| 10K / 15 s / 362 | 0.18 s | 20.28 s | 1.67 s | **24.01 s** | 100,408 MB | -All three requests finish faster than playback. Dense FA on the same weights -and topology takes 3.77 / 9.84 / 18.45 s (`sglang generate`, stage sum): it is -competitive at 5 s, and VSA-H3 pulls ahead from 10 s on. At 5 s, -TP2 + Ulysses2 (3.42 s, 62,290 MB), FSDP + Ulysses4 (3.42 s, 50,984 MB), and -online `--quantization fp8` (2.93 s, 64,204 MB) trade a little latency for -peak memory. +Over all 72 requests the E2E p50 / p90 / p95 is 12.94 / 23.94 / 24.08 s, and +every MP4 carries 1344×768 H.264 at 24 fps with a 32 kHz stereo AAC track. ### VDN-H3 on B200 diff --git a/docs/docs/sglang-diffusion/attention_backends.mdx b/docs/docs/sglang-diffusion/attention_backends.mdx index 62b39183a4..d54f4fac75 100644 --- a/docs/docs/sglang-diffusion/attention_backends.mdx +++ b/docs/docs/sglang-diffusion/attention_backends.mdx @@ -87,7 +87,7 @@ For SGLang-native pipelines, the CLI accepts the lowercase names of `AttentionBa `video_sparse_attn_h3` `VIDEO_SPARSE_ATTN_H3` - Video Sparse Attention for MiniMax-H3 / FastH3 (VSA-H3). In-tree Triton block-sparse kernel (SM90 / SM100 / SM103); no external package. Configure via --attention-backend-config. + Video Sparse Attention for MiniMax-H3 / FastH3 (VSA-H3). In-tree kernels, no external package: FastVideo's native tcgen05 block-sparse forward on SM100 / SM103, the Triton tile-64 kernel on SM90. Configure via --attention-backend-config. `hybrid_window_attn_h3` @@ -288,8 +288,8 @@ end-to-end latency on the target model and resolution before deployment. `VSA_sparsity` `float` - Fraction of video tiles excluded from the top-k selection (0.0 - 1.0). `0.9` is the FastH3 trained policy. - `0.9` + Fraction of video tiles excluded from the top-k selection (0.0 - 1.0). FastH3 defaults to its trained `0.8`. + `0.8` (FastH3), `0.9` (base H3) `vsa_mode` @@ -703,7 +703,7 @@ Hybrid window attention constraints: No ❌ ❌ - CUDA-only (SM90 / SM100 / SM103). In-tree Triton kernel, no external dependency. Configure via --attention-backend-config. + CUDA-only (SM90 / SM100 / SM103). In-tree kernels (native tcgen05 forward on Blackwell, Triton on Hopper), no external dependency. Configure via --attention-backend-config. sla_attn diff --git a/docs/docs/sglang-diffusion/fused_kernels.mdx b/docs/docs/sglang-diffusion/fused_kernels.mdx index a96f028cdf..206bb39a91 100644 --- a/docs/docs/sglang-diffusion/fused_kernels.mdx +++ b/docs/docs/sglang-diffusion/fused_kernels.mdx @@ -101,10 +101,11 @@ These fusion families mount under both `quality="lossless"` and | SANA-Video BF16-input linear attention | Keeps the first linear-attention GEMM's inputs in BF16 with FP32 accumulation/output; the second GEMM remains FP32 | | FLUX-family VAE fast paths | Channels-last decode, GroupNorm(+SiLU), upsample, and attention replacements for FLUX.2 and AutoencoderKL-based FLUX.1, Z-Image, and SD3 pipelines | | Wan VAE RMSNorm + SiLU | Replaces the channel-first RMSNorm/SiLU chain while keeping the decode in `channels_last_3d` | +| MiniMax-H3 VAE decoder fast path | Per-head fp32 RMSNorm, rounded to the activation dtype, and NeoX RoPE in one launch over the strided Q/K views of the ViT decoder's QKV buffer; cuDNN SDPA for the block's attention | ## Kernel inventory -45 operators are registered in the kernel registry across 51 implementations (some operators carry several backends). Backends are named by provenance, not device: `KDA` identifies Kernel Design Agents implementations, `JIT` compiles under nvcc *and* hipcc, `TRITON` identifies Triton sources, `CUTE_DSL` needs CUTLASS, `FLYDSL` is ROCm gfx950 only, and `AOT` comes from the `sgl_kernel` wheel. Per-operator capability metadata determines which devices can load each implementation. +46 operators are registered in the kernel registry across 52 implementations (some operators carry several backends). Backends are named by provenance, not device: `KDA` identifies Kernel Design Agents implementations, `JIT` compiles under nvcc *and* hipcc, `TRITON` identifies Triton sources, `CUTE_DSL` needs CUTLASS, `FLYDSL` is ROCm gfx950 only, and `AOT` comes from the `sgl_kernel` wheel. Per-operator capability metadata determines which devices can load each implementation. ### Normalization @@ -181,6 +182,7 @@ this path. This kernel does not enable model-level BCG support for FLUX.2. | --- | --- | --- | | `sparse_linear_attn_fwd` | Triton | block-map, compression, and forward for sparse linear attention | | `bigdn` | Triton | Sana-WM bidirectional gated delta-net; the chunkwise form splits phase A along the KV and Z streams so two blocks stay resident per SM, and stores `(I - P)` so phase B's MMA folds the identity-add in | +| `vsa_block_sparse_sm100` | JIT CUDA | FastVideo's warp-specialized tcgen05 block-sparse VSA forward for 64-token tiles (sm_100a / sm_103a); a CTA owns two adjacent query tiles with their own key lists. VSA-H3 uses it on Blackwell and keeps the Triton tile-64 kernel elsewhere | ### MoE routing diff --git a/docs/docs/sglang-diffusion/performance-optimization.mdx b/docs/docs/sglang-diffusion/performance-optimization.mdx index 6ed9f386ac..b0e097d655 100644 --- a/docs/docs/sglang-diffusion/performance-optimization.mdx +++ b/docs/docs/sglang-diffusion/performance-optimization.mdx @@ -253,7 +253,7 @@ Warmup and breakable CUDA graph (BCG) solve different problems. `--warmup-mode r | The model does not fit while both DiT stages are resident | Wan2.2-T2V-A14B, Wan2.2-I2V-A14B, LingBot-Video-MoE | Start with `--dit-layerwise-offload` | Verify peak GPU memory first, then measure the transfer overhead. Add warmup separately if first-run compilation is also material. | | Dynamic masks, metadata, or image conditions make graph replay ineffective | Qwen-Image-Edit family, FireRed-Image-Edit | Keep eager execution as the baseline | Profile before attempting BCG. Dynamic per-step host work can outweigh graph replay savings, and unsupported pipelines fall back to eager execution. | | Kernels or collectives already dominate a many-step workload | Qwen-Image, FLUX.1-dev, FLUX.2-dev, LTX-2, Wan2.1 14B, HunyuanVideo, MOVA | Warm the baseline, then profile before enabling another lever | A small host gap limits the benefit available from BCG. Prioritize kernels, attention, parallelism, or residency according to the trace. | -| A one-shot or very few-step workload cannot amortize warmup | FastVideo-FastH3 (4-step) | Start with `--warmup-mode off` | Compare end-to-end latency including warmup. Use request warmup only when intentionally separating cold-start setup from the measured request. | +| A one-shot or very few-step workload cannot amortize warmup | FastVideo-FastH3 (8-step) | Start with `--warmup-mode off` | Compare end-to-end latency including warmup. Use request warmup only when intentionally separating cold-start setup from the measured request. | The example assignments come from single-GPU NVIDIA B300/GB300 profiles and are directional rather than an exhaustive compatibility list. A model can match more than one row as resolution, frame count, step count, parallelism, or GPU type changes; the measured bottleneck takes precedence over the model name. diff --git a/docs/src/snippets/diffusion/model-catalog.jsx b/docs/src/snippets/diffusion/model-catalog.jsx index 0a4dc67152..74184acc60 100644 --- a/docs/src/snippets/diffusion/model-catalog.jsx +++ b/docs/src/snippets/diffusion/model-catalog.jsx @@ -219,9 +219,9 @@ export const DiffusionModelCatalog = ({ category }) => { { name: "FastH3", modelIds: [ - "FastVideo/FastVideo-FastH3-4-step-Preview-v1-VSA-DataFree", + "FastVideo/FastVideo-FastH3-8-Step-V2", ], - cookbook: "/cookbook/diffusion/MiniMax/MiniMax-H3#6-fasth3-4-step-distilled-preview", + cookbook: "/cookbook/diffusion/MiniMax/MiniMax-H3#6-fasth3-8-step-distilled", }, { name: "VDN-H3", diff --git a/python/sglang/kernels/jit/csrc/diffusion/vsa_block_sparse_sm100.cuh b/python/sglang/kernels/jit/csrc/diffusion/vsa_block_sparse_sm100.cuh new file mode 100644 index 0000000000..681f4758db --- /dev/null +++ b/python/sglang/kernels/jit/csrc/diffusion/vsa_block_sparse_sm100.cuh @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: Apache-2.0 +#include +#include + +#include + +#include +#include + +#define VSA_BHSD true +#include "vsa_block_sparse_sm100/block_sparse_launch_sm100a.cuh" + +namespace sglang { + +/** + * \brief FastVideo's warp-specialized tcgen05 block-sparse attention forward for 64-token + * tiles (sm_100a / sm_103a). + * + * \param q, k, v, out [B, H, S, 128] bf16, contiguous; S == num_blocks * 64, num_blocks even. + * \param q2k_idx [B * H * num_blocks, max_kv] int32 key-tile lists (valid below q2k_num). + * \param q2k_num [B * H * num_blocks] int32 per-query-tile list lengths (0 allowed). + * \param block_sizes [num_blocks] int32 valid tokens per key tile. + */ +struct VsaBlockSparseSm100Kernel { + static void + run(const tvm::ffi::TensorView q, + const tvm::ffi::TensorView k, + const tvm::ffi::TensorView v, + const tvm::ffi::TensorView q2k_idx, + const tvm::ffi::TensorView q2k_num, + const tvm::ffi::TensorView block_sizes, + tvm::ffi::TensorView out, + double sm_scale) { + using namespace host; + + auto B = SymbolicSize{"batch"}; + auto H = SymbolicSize{"heads"}; + auto S = SymbolicSize{"seq_pad"}; + auto R = SymbolicSize{"rows"}; + auto N = SymbolicSize{"num_blocks"}; + auto MAXKV = SymbolicSize{"max_kv"}; + auto device = SymbolicDevice{}; + device.set_options(); + + TensorMatcher({B, H, S, HEAD_DIM}) + .with_dtype() + .with_device(device) + .verify(q) + .verify(k) + .verify(v) + .verify(out); + TensorMatcher({R, MAXKV}).with_dtype().with_device(device).verify(q2k_idx); + TensorMatcher({R}).with_dtype().with_device(device).verify(q2k_num); + TensorMatcher({N}).with_dtype().with_device(device).verify(block_sizes); + + const int64_t num_blocks = N.unwrap(); + CHECK_HOST(num_blocks % 2 == 0) << "num_blocks must be even, got " << num_blocks; + CHECK_HOST(S.unwrap() == num_blocks * BLOCK) << "seq_pad " << S.unwrap() << " != num_blocks * " << BLOCK; + CHECK_HOST(R.unwrap() == B.unwrap() * H.unwrap() * num_blocks) << "q2k rows must be batch * heads * num_blocks"; + + BlockSparseVsaArgs a{}; + a.q = static_cast(q.data_ptr()); + a.k = static_cast(k.data_ptr()); + a.v = static_cast(v.data_ptr()); + a.v_t = nullptr; + a.o = static_cast<__nv_bfloat16*>(out.data_ptr()); + a.lse = nullptr; + a.q2k_idx = static_cast(q2k_idx.data_ptr()); + a.q2k_num = static_cast(q2k_num.data_ptr()); + a.variable_block_sizes = static_cast(block_sizes.data_ptr()); + a.batch = static_cast(B.unwrap()); + a.num_heads = static_cast(H.unwrap()); + a.seqlen = static_cast(S.unwrap()); + a.head_dim = HEAD_DIM; + a.num_blocks = static_cast(num_blocks); + a.max_kv = static_cast(MAXKV.unwrap()); + a.sm_scale = static_cast(sm_scale); + + const DLDevice dev = device.unwrap(); + auto stream = static_cast(::TVMFFIEnvGetStream(dev.device_type, dev.device_id)); + CHECK_CUDA(launch_block_sparse_sm100a(a, stream)) << "vsa_block_sparse_sm100 launch"; + } +}; + +} // namespace sglang diff --git a/python/sglang/kernels/jit/csrc/diffusion/vsa_block_sparse_sm100/block_sparse_kernel_sm100a.cuh b/python/sglang/kernels/jit/csrc/diffusion/vsa_block_sparse_sm100/block_sparse_kernel_sm100a.cuh new file mode 100644 index 0000000000..09f518aad7 --- /dev/null +++ b/python/sglang/kernels/jit/csrc/diffusion/vsa_block_sparse_sm100/block_sparse_kernel_sm100a.cuh @@ -0,0 +1,1083 @@ +// Copied and adapted from: https://github.com/hao-ai-lab/FastVideo +// (fastvideo-kernel/csrc/attention/block_sparse_kernel_sm100a.cuh, Apache-2.0). Inference-only +// forward; the sm_103a device pass is admitted alongside sm_100a. + +// block_sparse_kernel_sm100a.cuh -- VSA block-sparse FMHA forward (per-q-block top-k), sm_100a. +// Warp-specialized: load / MMA (tcgen05) / softmax / correction / epilogue / scheduler. +// Writes O and, when asked, the log-sum-exp the backward consumes. +// +// Generated (comments stripped). Do not edit by hand. +#ifndef BLOCK_SPARSE_VSA_KERNEL_SM100A_CUH +#define BLOCK_SPARSE_VSA_KERNEL_SM100A_CUH + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "primitives.cuh" + +#ifndef VSA_BLK128 +#define VSA_BLK128 false +#endif +#ifndef VSA_BHSD +#define VSA_BHSD false +#endif + +// BLK128 is a file-scope constexpr, not a template parameter, so the blk64 and blk128 builds +// would instantiate the SAME kernel symbol with DIFFERENT bodies -- an ODR violation the linker +// resolves by silently keeping one. A config-named namespace keeps the two builds' symbols +// distinct so both can live in one extension. +#if VSA_BLK128 +#define VSA_NAMESPACE vsa_blk128 +#else +#define VSA_NAMESPACE vsa_blk64 +#endif +namespace VSA_NAMESPACE { + +constexpr bool BLK128 = VSA_BLK128; + +#ifndef VSA_DEFER_ROWSUM +#define VSA_DEFER_ROWSUM (!VSA_BLK128) +#endif +constexpr bool DEFER_ROWSUM = VSA_DEFER_ROWSUM; + +constexpr int BLOCK = BLK128 ? 128 : 64; +constexpr int M_TILE = BLOCK; +constexpr int M_TILES_PER_CTA = 2; +constexpr int K_TILE = 256 / (BLK128 ? 2 : 1); +constexpr int BLOCKS_PER_KTILE = K_TILE / BLOCK; +constexpr int KV_HALF = K_TILE / 2; +constexpr int HEAD_DIM = 128; + +constexpr int SUB_COLS_BF16 = 64; +constexpr int SUB_COLS_BYTES = SUB_COLS_BF16 * (int)sizeof(__nv_bfloat16); +constexpr int Q_SUBTILES = HEAD_DIM / SUB_COLS_BF16; +constexpr int K_SUBTILES = HEAD_DIM / SUB_COLS_BF16; +constexpr int V_SUBTILES = (BLK128 ? K_TILE : KV_HALF) / SUB_COLS_BF16; +constexpr int Q_SUB_COLS_BYTES = M_TILE * SUB_COLS_BYTES; +constexpr int K_SUB_COLS_BYTES = K_TILE * SUB_COLS_BYTES; +constexpr int Q_TILE_BYTES = Q_SUBTILES * Q_SUB_COLS_BYTES; + +constexpr int KV_RING_SLOT_BYTES = 32 * 1024; +constexpr int SLOTS_PER_KV_TILE = BLK128 ? 1 : 2; + +constexpr int NUM_KV_STAGES = BLK128 ? 3 : 4; + +constexpr int V_BLK_BYTES = HEAD_DIM * SUB_COLS_BYTES; + +constexpr int S_COLS = 128; +constexpr int K_ATOMS_PER_TILE = SUB_COLS_BF16 / 16; +constexpr int SPLIT_P_N = S_COLS / 4 * 3; +constexpr int SPLIT_P_ATOM = SPLIT_P_N / 16; +constexpr int SPLIT_P_COL = SPLIT_P_N / 2; +constexpr int EX2_FRG_PAIRS = 16; +constexpr int EX2_FRG_CNT = S_COLS / 32; +constexpr int EX2_FREQ = 16; +constexpr int EX2_RES = 4; + +constexpr int O_COLS = HEAD_DIM; +constexpr int TMEM_TOTAL = 512; + +constexpr int STATS = BLK128 ? M_TILE : 2 * M_TILE; +constexpr int STAT_REGIONS = BLK128 ? 2 : 3; + +constexpr int W_CORR0 = 8, W_MMA = 12, W_EPI = 13, W_LOAD = 14, W_SCHED = 15; +constexpr int N_WARPS = 16; +constexpr int CLC_STAGES = 2; + +extern __shared__ __align__(1024) uint8_t fmha_smem[]; + +union SmemDescPair { + uint64_t u64; + uint2 w; +}; + +__device__ __forceinline__ void desc_add_lo(SmemDescPair& d, uint32_t inc) { + asm volatile( + "{\n\t" + ".reg .b32 lo, hi;\n\t" + "mov.b64 {lo, hi}, %0;\n\t" + "add.u32 lo, lo, %1;\n\t" + "mov.b64 %0, {lo, hi};\n\t" + "}" + : "+l"(d.u64) + : "r"(inc)); +} + +__device__ __forceinline__ void tcgen05_mma_ws_f16_ss_1sm_lead(uint32_t lead, + uint32_t tmem_d, uint64_t desc_a, uint64_t desc_b, uint32_t idesc, + bool enable_input_d) { + asm volatile( + "{\n\t" + ".reg .pred p, q;\n\t" + "setp.ne.b32 q, %0, 0;\n\t" + "setp.ne.b32 p, %4, 0;\n\t" + "@q tcgen05.mma.ws.cta_group::1.kind::f16 [%1], %2, %3, %5, p, 0;\n\t" + "}\n" + :: "r"(lead), "r"(tmem_d), "l"(desc_a), "l"(desc_b), + "r"(enable_input_d ? 1u : 0u), "r"(idesc)); +} +__device__ __forceinline__ void tcgen05_mma_ws_f16_ts_1sm_lead(uint32_t lead, + uint32_t tmem_d, uint32_t tmem_a, uint64_t desc_b, uint32_t idesc, + bool enable_input_d) { + asm volatile( + "{\n\t" + ".reg .pred p, q;\n\t" + "setp.ne.b32 q, %0, 0;\n\t" + "setp.ne.b32 p, %4, 0;\n\t" + "@q tcgen05.mma.ws.cta_group::1.kind::f16 [%1], [%2], %3, %5, p, 0;\n\t" + "}\n" + :: "r"(lead), "r"(tmem_d), "r"(tmem_a), "l"(desc_b), + "r"(enable_input_d ? 1u : 0u), "r"(idesc)); +} + +struct WorkItem { + int sample; + int head; + int mtile0; + int mtile1; + int global_mtile0; + int global_mtile1; + // Shared trip count for the CTA's q-tile pair: max of the two tiles' own + // counts, floored at 1. Every warp role derives its loop bounds from this + // one value, so the producer/consumer mbarrier pairing stays symmetric for + // ANY per-tile counts (including 0): a tile shorter than the pair max runs + // its extra iterations against clamped (valid, in-bounds) KV blocks and + // masks them to -inf via the per-tile count below, contributing nothing. + int num_kv_blocks; + // Each q-tile's OWN q2k_num. Used for (a) the q2k_idx window clamp in the + // load warp and (b) the vbs-threshold masking in the softmax warps. The + // pre-fix kernel used q2k_num[global_mtile0] for BOTH tiles, silently + // corrupting one tile of any pair whose rows have different counts. + int num_kv_blocks_mt[2]; +}; +template +__device__ __forceinline__ WorkItem decode_workitem( + int workitem_id, int num_heads, int num_blocks, int packed_mtiles_per_seq, + unsigned long long magic0, unsigned long long magic1, unsigned long long magic2, const int* q2k_num) { + WorkItem it; + const int per_sample = num_heads * packed_mtiles_per_seq; + it.sample = (int)fdiv((unsigned)workitem_id, magic0); + const int rem = workitem_id - it.sample * per_sample; + int p; + if constexpr (Q_RASTER) { + it.head = (int)fdiv((unsigned)rem, magic2); + p = rem - it.head * packed_mtiles_per_seq; + } else { + p = (int)fdiv((unsigned)rem, magic1); + it.head = rem - p * num_heads; + } + it.mtile0 = 2 * p; + it.mtile1 = 2 * p + 1; + it.global_mtile0 = (it.sample * num_heads + it.head) * num_blocks + it.mtile0; + it.global_mtile1 = it.global_mtile0 + 1; + it.num_kv_blocks_mt[0] = q2k_num[it.global_mtile0]; + it.num_kv_blocks_mt[1] = q2k_num[it.global_mtile1]; + // Floor of 1: a pair whose rows are BOTH empty still walks one K-tile so no + // mbarrier wait is left without its arrive (the load/MMA/softmax/correction + // pipeline has a fixed per-iteration handshake); the per-tile counts mask + // that K-tile completely, so such rows produce exactly-zero output. + it.num_kv_blocks = max(max(it.num_kv_blocks_mt[0], it.num_kv_blocks_mt[1]), 1); + return it; +} + +template +__global__ void __cluster_dims__(1, 1, 1) __launch_bounds__(N_WARPS * 32, 1) +fmha_context_bf16_gen_kernel(const __grid_constant__ CUtensorMap tmap_q, + const __grid_constant__ CUtensorMap tmap_k, const __grid_constant__ CUtensorMap tmap_v_t, + const __grid_constant__ CUtensorMap tmap_v, + const __grid_constant__ CUtensorMap tmap_o, int seqlen, + int num_heads, float scale_log2, int num_samples, int num_blocks, + int packed_mtiles_per_seq, int max_kv, + unsigned long long magic0, unsigned long long magic1, unsigned long long magic2, + const int* __restrict__ q2k_idx, const int* __restrict__ q2k_num, + const int* __restrict__ variable_block_sizes, + float* __restrict__ lse_out) { +// Multi-arch builds: torch's cmake appends -gencode for EVERY entry of +// TORCH_CUDA_ARCH_LIST to this TU on top of the pinned compute_100a pass, and +// tcgen05/setmaxnreg do not exist outside sm_100a -- ptxas rejects the sm_120a +// (or plain sm_100) pass outright. Keep the body only where it can compile: +// the host pass (no __CUDA_ARCH__, needed for launch plumbing) and the +// sm_100a device pass (arch 1000 WITH the family-specific feature set that the +// "a" suffix defines). Every other device pass gets an empty stub; the Python +// is_supported() / host launcher never dispatch here off sm_100, so the stub +// is unreachable at runtime. +#if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ == 1000 && defined(__CUDA_ARCH_FEAT_SM100_ALL)) || \ + (__CUDA_ARCH__ == 1030 && defined(__CUDA_ARCH_FEAT_SM103_ALL)) + + const int total_workitems = num_samples * num_heads * packed_mtiles_per_seq; + + uint8_t* sQ0 = fmha_smem; + uint8_t* sQ1 = sQ0 + Q_TILE_BYTES; + uint8_t* sQ[2] = { sQ0, sQ1 }; + uint8_t* sKV = sQ1 + Q_TILE_BYTES; + __nv_bfloat16* sO0 = reinterpret_cast<__nv_bfloat16*>(sKV + NUM_KV_STAGES * KV_RING_SLOT_BYTES); + __nv_bfloat16* sO1 = sO0 + M_TILE * HEAD_DIM; + __nv_bfloat16* sO_bufs[2] = { sO0, sO1 }; + uint64_t* full_bar = reinterpret_cast(reinterpret_cast(sO1) + M_TILE * HEAD_DIM * sizeof(__nv_bfloat16)); + uint64_t* empty_bar= full_bar + NUM_KV_STAGES; + uint64_t* full_bar_q = empty_bar + NUM_KV_STAGES; + uint64_t* empty_bar_q = full_bar_q + 2; + uint64_t* full_bar_spo = empty_bar_q + 2; + uint64_t* empty_bar_spo = full_bar_spo + 2; + uint64_t* full_bar_o_acc = empty_bar_spo + 2; + uint64_t* full_bar_alpha = full_bar_o_acc + 2; + uint64_t* full_bar_l = full_bar_alpha + 2; + uint64_t* full_bar_p_last = full_bar_l + 2; + uint64_t* empty_bar_alpha_and_l = full_bar_p_last + 2; + uint64_t* full_bar_o_epi = empty_bar_alpha_and_l + 2; + uint64_t* empty_bar_o_epi = full_bar_o_epi + 2; + uint64_t* clc_full = empty_bar_o_epi + 2; + uint64_t* clc_empty = clc_full + CLC_STAGES; + uint32_t* clc_response = reinterpret_cast( + (reinterpret_cast(clc_empty + CLC_STAGES) + 15u) & ~uintptr_t(15u)); + uint32_t* tmem_slot = clc_response + CLC_STAGES * 4; + float* alpha_and_l_smem = reinterpret_cast(tmem_slot + 2); + + const int tid = threadIdx.x, warp_id = tid >> 5, lane = tid & 31; + + if (warp_id == 0) { + tcgen05_alloc<1>(smem_ptr_u32(tmem_slot), TMEM_TOTAL); + tcgen05_relinquish_alloc_permit<1>(); + } + __syncthreads(); + const uint32_t tmem_base = *tmem_slot; + + if (tid == 0) { + #pragma unroll + for (int s = 0; s < NUM_KV_STAGES; ++s) { + mbarrier_init(smem_ptr_u32(&full_bar[s]), 1); + mbarrier_init(smem_ptr_u32(&empty_bar[s]), 1); + } + for (int i = 0; i < 2; ++i) { + mbarrier_init(smem_ptr_u32(&full_bar_q[i]), 1); + mbarrier_init(smem_ptr_u32(&empty_bar_q[i]), 1); + mbarrier_init(smem_ptr_u32(&full_bar_l[i]), 128); + mbarrier_init(smem_ptr_u32(&full_bar_spo[i]), 1); + mbarrier_init(smem_ptr_u32(&empty_bar_spo[i]), 256); + mbarrier_init(smem_ptr_u32(&full_bar_o_acc[i]), 1); + mbarrier_init(smem_ptr_u32(&full_bar_alpha[i]), 128); + mbarrier_init(smem_ptr_u32(&empty_bar_alpha_and_l[i]), 128); + mbarrier_init(smem_ptr_u32(&full_bar_p_last[i]), 128); + mbarrier_init(smem_ptr_u32(&full_bar_o_epi[i]), 128); + mbarrier_init(smem_ptr_u32(&empty_bar_o_epi[i]), 1); + } + if constexpr (USE_CLC) { + #pragma unroll + for (int s = 0; s < CLC_STAGES; ++s) { + mbarrier_init(smem_ptr_u32(&clc_full[s]), 1); + mbarrier_init(smem_ptr_u32(&clc_empty[s]), N_WARPS); + } + #pragma unroll + for (int i = 0; i < CLC_STAGES * 4; ++i) + clc_response[i] = 0; + } + } + fence_mbarrier_init_release_cluster(); + __syncthreads(); + + if (warp_id == W_LOAD) { + setmaxnreg_dec<48>(); + + EmptyPhaseTracker kv_empty_ph; + EmptyPhaseTracker<1> q_empty_ph; + [[maybe_unused]] int clc_stage = 0; + [[maybe_unused]] uint32_t clc_phase = 0; + int workitem_id = (int)blockIdx.x; + while (true) { + const WorkItem it = decode_workitem(workitem_id, num_heads, num_blocks, packed_mtiles_per_seq, magic0, magic1, magic2, q2k_num); + const int q_start = it.sample * seqlen; + const int k_start = it.sample * seqlen; + const int global_mtile[2] = { it.global_mtile0, it.global_mtile1 }; + const int mtile[2] = { it.mtile0, it.mtile1 }; + const int num_k_tiles = (it.num_kv_blocks + BLOCKS_PER_KTILE - 1) / BLOCKS_PER_KTILE; + + int window_start[2] = { -(1 << 30), -(1 << 30) }; + int kv_block_id_cache[2] = { 0, 0 }; + auto get_kv_block_id = [&](int mtile_idx, int j) -> int { + if (j >= window_start[mtile_idx] + 32) { + window_start[mtile_idx] = j & ~31; + // Clamp into THIS tile's own row of q2k_idx (the pair may run more + // iterations than this tile has blocks -- see WorkItem). Positions + // past the tile's count re-read its last valid entry; an empty tile + // never dereferences its row at all and loads block 0 instead. Both + // are fully masked by the per-tile vbs thresholds, so only the + // load address safety matters here. + const int cnt = it.num_kv_blocks_mt[mtile_idx]; + const int idx = max(0, min(window_start[mtile_idx] + lane, cnt - 1)); + kv_block_id_cache[mtile_idx] = + (cnt > 0) ? q2k_idx[global_mtile[mtile_idx] * max_kv + idx] : 0; + } + return __shfl_sync(0xffffffffu, kv_block_id_cache[mtile_idx], j & 31); + }; + + auto load_k_oneslot = [&](int mtile_idx, int ktile_idx, int s) { + const int kv_stage = kv_empty_ph.get_stage(); + + mbarrier_wait_parity_suspend(smem_ptr_u32(&empty_bar[kv_stage]), kv_empty_ph.get_phase()); + kv_empty_ph.advance(); + + int kv_tok[BLOCKS_PER_KTILE]; + #pragma unroll + for (int blk = 0; blk < BLOCKS_PER_KTILE; ++blk) + kv_tok[blk] = k_start + get_kv_block_id(mtile_idx, ktile_idx * BLOCKS_PER_KTILE + blk) * BLOCK; + if (elect_one_sync()) { + mbarrier_arrive_expect_tx(smem_ptr_u32(&full_bar[kv_stage]), KV_RING_SLOT_BYTES); + if constexpr (BLK128) { + + if constexpr (BHSD) + tma_load_4d(smem_ptr_u32((sKV + kv_stage * KV_RING_SLOT_BYTES)), &tmap_k, smem_ptr_u32(&full_bar[kv_stage]), + 0, kv_tok[0] - k_start, 0, it.sample * num_heads + it.head); + else + tma_load_3d(smem_ptr_u32((sKV + kv_stage * KV_RING_SLOT_BYTES)), &tmap_k, smem_ptr_u32(&full_bar[kv_stage]), + 0, kv_tok[0], it.head * K_SUBTILES); + } else { + #pragma unroll + for (int blk = 0; blk < BLOCKS_PER_KTILE; ++blk) + if constexpr (BHSD) + tma_load_4d(smem_ptr_u32((sKV + kv_stage * KV_RING_SLOT_BYTES) + blk * BLOCK * SUB_COLS_BYTES), + &tmap_k, smem_ptr_u32(&full_bar[kv_stage]), + 0, kv_tok[blk] - k_start, s, it.sample * num_heads + it.head); + else + tma_load_3d(smem_ptr_u32((sKV + kv_stage * KV_RING_SLOT_BYTES) + blk * BLOCK * SUB_COLS_BYTES), + &tmap_k, smem_ptr_u32(&full_bar[kv_stage]), + 0, kv_tok[blk], it.head * K_SUBTILES + s); + } + } + }; + auto load_v_oneslot = [&](int mtile_idx, int ktile_idx, int p) { + const int kv_stage = kv_empty_ph.get_stage(); + + mbarrier_wait_parity_suspend(smem_ptr_u32(&empty_bar[kv_stage]), kv_empty_ph.get_phase()); + kv_empty_ph.advance(); + + if constexpr (BLK128) { + + const int kv_tok0 = k_start + get_kv_block_id(mtile_idx, ktile_idx) * BLOCK; + if (elect_one_sync()) { + mbarrier_arrive_expect_tx(smem_ptr_u32(&full_bar[kv_stage]), KV_RING_SLOT_BYTES); + + if constexpr (BHSD) + tma_load_4d(smem_ptr_u32((sKV + kv_stage * KV_RING_SLOT_BYTES)), &tmap_v, smem_ptr_u32(&full_bar[kv_stage]), + 0, kv_tok0 - k_start, 0, it.sample * num_heads + it.head); + else + tma_load_3d(smem_ptr_u32((sKV + kv_stage * KV_RING_SLOT_BYTES)), &tmap_v, smem_ptr_u32(&full_bar[kv_stage]), + 0, kv_tok0, it.head * K_SUBTILES); + } + } else { + int kv_tok[2]; + #pragma unroll + for (int h = 0; h < 2; ++h) + kv_tok[h] = k_start + get_kv_block_id(mtile_idx, ktile_idx * BLOCKS_PER_KTILE + 2 * h + p) * BLOCK; + if (elect_one_sync()) { + mbarrier_arrive_expect_tx(smem_ptr_u32(&full_bar[kv_stage]), KV_RING_SLOT_BYTES); + + #pragma unroll + for (int h = 0; h < 2; ++h) { + #pragma unroll + for (int s2 = 0; s2 < K_SUBTILES; ++s2) { + const uint32_t dst = smem_ptr_u32((sKV + kv_stage * KV_RING_SLOT_BYTES) + + h * V_BLK_BYTES + s2 * (BLOCK * SUB_COLS_BYTES)); + if constexpr (BHSD) + tma_load_4d(dst, &tmap_v, smem_ptr_u32(&full_bar[kv_stage]), + 0, kv_tok[h] - k_start, s2, it.sample * num_heads + it.head); + else + tma_load_3d(dst, &tmap_v, smem_ptr_u32(&full_bar[kv_stage]), + 0, kv_tok[h], it.head * K_SUBTILES + s2); + } + } + } + } + }; + auto load_k = [&](int mtile_idx, int ktile_idx) { + load_k_oneslot(mtile_idx, ktile_idx, 0); + if constexpr (!BLK128) load_k_oneslot(mtile_idx, ktile_idx, 1); + }; + auto load_v = [&](int mtile_idx, int ktile_idx) { + load_v_oneslot(mtile_idx, ktile_idx, 0); + if constexpr (!BLK128) load_v_oneslot(mtile_idx, ktile_idx, 1); + }; + + load_k(0, 0); load_k(1, 0); + + #pragma unroll + for (int m = 0; m < M_TILES_PER_CTA; ++m) { + mbarrier_wait_parity_suspend(smem_ptr_u32(&empty_bar_q[m]), q_empty_ph.get_phase()); + const int tok0 = q_start + mtile[m] * BLOCK; + if (elect_one_sync()) { + mbarrier_arrive_expect_tx(smem_ptr_u32(&full_bar_q[m]), Q_TILE_BYTES); + + tma_load_4d(smem_ptr_u32(sQ[m]), &tmap_q, smem_ptr_u32(&full_bar_q[m]), + 0, BHSD ? it.sample * num_heads + it.head : it.head, + BHSD ? tok0 - q_start : tok0, 0); + } + } + q_empty_ph.advance(); + for (int ktile_idx = 0; ktile_idx + 1 < num_k_tiles; ++ktile_idx) { + load_v(0, ktile_idx); load_k(0, ktile_idx + 1); + load_v(1, ktile_idx); load_k(1, ktile_idx + 1); + } + load_v(0, num_k_tiles - 1); load_v(1, num_k_tiles - 1); + + if constexpr (USE_CLC) { + ClcTileInfo next = clc_fetch_next_tile<1, 1, ClcRasterOrder::AlongN, 1, true>( + clc_full, clc_empty, clc_response, clc_stage, clc_phase, elect_one_sync()); + clc_fetch_next_tile_advance(clc_stage, clc_phase); + if (!next.valid) break; + workitem_id = next.n_tile; + } else { + workitem_id += gridDim.x; + if (workitem_id >= total_workitems) break; + } + } + } + else if (warp_id == W_MMA) { + setmaxnreg_dec<48>(); + + const uint32_t lead = elect_one_sync() ? 1u : 0u; + constexpr uint32_t DESC_SBO = 1024, DESC_LBO = 16; + + const uint32_t idesc_qk = make_idesc_bf16_f32(M_TILE, K_TILE, false, false); + const uint32_t idesc_pv = make_idesc_bf16_f32(M_TILE, BLK128 ? HEAD_DIM : 2 * HEAD_DIM, + false, true); + + constexpr uint32_t DESC_LBO_MN = (uint32_t)((BLK128 ? K_TILE : BLOCK) * SUB_COLS_BF16 * 2); + const uint64_t desc_v0 = build_smem_desc_blackwell(smem_ptr_u32(sKV), DESC_SBO, DESC_LBO_MN, + SmemSwizzleBlackwell::B128); + constexpr uint32_t PV_DESC_STEP = (uint32_t)((16 * SUB_COLS_BF16 * 2) >> 4); + const uint64_t desc_q0 = build_smem_desc_blackwell(smem_ptr_u32(sQ0), DESC_SBO, DESC_LBO, SmemSwizzleBlackwell::B128); + const uint64_t desc_kv0 = build_smem_desc_blackwell(smem_ptr_u32(sKV), DESC_SBO, DESC_LBO, SmemSwizzleBlackwell::B128); + + constexpr uint64_t KV_DESC_DELTA = KV_RING_SLOT_BYTES >> 4; + constexpr uint64_t Q_SUB_DELTA = Q_SUB_COLS_BYTES >> 4; + constexpr uint64_t Q_MTILE_DESC_DELTA = Q_TILE_BYTES >> 4; + + PhaseTracker kv_ph; + PhaseTracker<1> q_ph; + PhaseTracker<1> spo_ph; + [[maybe_unused]] int clc_stage = 0; + [[maybe_unused]] uint32_t clc_phase = 0; + int workitem_id = (int)blockIdx.x; + while (true) { + const WorkItem it = decode_workitem(workitem_id, num_heads, num_blocks, packed_mtiles_per_seq, magic0, magic1, magic2, q2k_num); + const int num_k_tiles = (it.num_kv_blocks + BLOCKS_PER_KTILE - 1) / BLOCKS_PER_KTILE; + + auto bmm1 = [&](int i) { + const uint32_t s_tmem_addr = tmem_base + (uint32_t)(i * S_COLS); + const uint64_t da_base = desc_q0 + (uint64_t)i * Q_MTILE_DESC_DELTA; + + int slot = 0; + #pragma unroll + for (int s = 0; s < Q_SUBTILES; ++s) { + if (!BLK128 || s == 0) { + slot = kv_ph.get_stage(); + mbarrier_wait_parity(smem_ptr_u32(&full_bar[slot]), kv_ph.get_phase()); + kv_ph.advance(); + } + + const uint64_t da = da_base + (uint64_t)s * Q_SUB_DELTA; + const uint64_t db = desc_kv0 + (uint64_t)slot * KV_DESC_DELTA + + (BLK128 ? (uint64_t)s * (K_SUB_COLS_BYTES >> 4) : 0u); + #pragma unroll + for (int ki = 0; ki < K_ATOMS_PER_TILE; ++ki) { + const bool enable_d = (s != 0) || (ki != 0); + if constexpr (BLK128) + tcgen05_mma_f16_ss_lead(lead, s_tmem_addr, da + 2 * ki, db + 2 * ki, idesc_qk, enable_d); + else + tcgen05_mma_ws_f16_ss_1sm_lead(lead, s_tmem_addr, da + 2 * ki, db + 2 * ki, idesc_qk, enable_d); + } + if (!BLK128 || s == Q_SUBTILES - 1) + tcgen05_commit1_lead(lead, smem_ptr_u32(&empty_bar[slot])); + } + + tcgen05_commit1_lead(lead, smem_ptr_u32(&full_bar_spo[i])); + }; + + auto bmm2 = [&](int i, bool first_ktile, auto last_c) { + constexpr bool last_ktile = decltype(last_c)::value; + const uint32_t s_tmem_addr = tmem_base + (uint32_t)(i * S_COLS); + const uint32_t o_tmem_addr = tmem_base + (uint32_t)(2 * S_COLS + i * O_COLS); + + int slot = 0; + #pragma unroll + for (int p = 0; p < V_SUBTILES; ++p) { + SmemDescPair desc_bv; + if (!BLK128 || p == 0) { + slot = kv_ph.get_stage(); + mbarrier_wait_parity(smem_ptr_u32(&full_bar[slot]), kv_ph.get_phase()); + kv_ph.advance(); + } + if (!BLK128 || p == 0) { + desc_bv.u64 = desc_v0; + desc_bv.w.x += (uint32_t)(slot * (int)KV_DESC_DELTA); + } + const uint64_t dbV = desc_kv0 + (uint64_t)slot * KV_DESC_DELTA + + (BLK128 ? (uint64_t)p * (V_BLK_BYTES >> 4) : 0u); + #pragma unroll + for (int ki = 0; ki < K_ATOMS_PER_TILE; ++ki) { + const int a = p * K_ATOMS_PER_TILE + ki; + if constexpr (SPLIT_P) if (a == SPLIT_P_ATOM) { + mbarrier_wait_parity(smem_ptr_u32(&full_bar_p_last[i]), spo_ph.get_phase()); + } + const bool accumulate = (!first_ktile) || (a != 0); + if constexpr (BLK128) { + tcgen05_mma_f16_ts_1sm_lead(lead, o_tmem_addr, s_tmem_addr + (uint32_t)(a * 8), desc_bv.u64, idesc_pv, accumulate); + desc_add_lo(desc_bv, PV_DESC_STEP); + } else { + tcgen05_mma_ws_f16_ts_1sm_lead(lead, o_tmem_addr, s_tmem_addr + (uint32_t)(a * 8), desc_bv.u64, idesc_pv, accumulate); + desc_add_lo(desc_bv, PV_DESC_STEP); + } + } + + if (!BLK128 || p == V_SUBTILES - 1) { + tcgen05_commit1_lead(lead, smem_ptr_u32(&empty_bar[slot])); + } + } + + if constexpr (last_ktile) { + tcgen05_commit1_lead(lead, smem_ptr_u32(&full_bar_o_acc[i])); + } + }; + + #pragma unroll + for (int i = 0; i < M_TILES_PER_CTA; ++i) { + mbarrier_wait_parity(smem_ptr_u32(&full_bar_q[i]), q_ph.get_phase()); + bmm1(i); + } + + for (int k = 0; k + 1 < num_k_tiles; ++k) { + #pragma unroll + for (int i = 0; i < M_TILES_PER_CTA; ++i) { + mbarrier_wait_parity(smem_ptr_u32(&empty_bar_spo[i]), spo_ph.get_phase()); + bmm2(i, (k == 0), std::false_type{}); + bmm1(i); + } + + spo_ph.advance(); + } + + #pragma unroll + for (int i = 0; i < M_TILES_PER_CTA; ++i) + tcgen05_commit1_lead(lead, smem_ptr_u32(&empty_bar_q[i])); + q_ph.advance(); + + #pragma unroll + for (int i = 0; i < M_TILES_PER_CTA; ++i) { + mbarrier_wait_parity(smem_ptr_u32(&empty_bar_spo[i]), spo_ph.get_phase()); + bmm2(i, (num_k_tiles == 1), std::true_type{}); + } + + spo_ph.advance(); + + if constexpr (USE_CLC) { + ClcTileInfo next = clc_fetch_next_tile<1, 1, ClcRasterOrder::AlongN, 1, true>( + clc_full, clc_empty, clc_response, clc_stage, clc_phase, elect_one_sync()); + clc_fetch_next_tile_advance(clc_stage, clc_phase); + if (!next.valid) break; + workitem_id = next.n_tile; + } else { + workitem_id += gridDim.x; + if (workitem_id >= total_workitems) break; + } + } + } + else if (warp_id == W_EPI) { + setmaxnreg_dec<48>(); + + PhaseTracker<1> full_o_ph; + + if (elect_one_sync()) { + #pragma unroll + for (int m = 0; m < M_TILES_PER_CTA; ++m) + mbarrier_arrive(smem_ptr_u32(&empty_bar_o_epi[m])); + } + [[maybe_unused]] int clc_stage = 0; + [[maybe_unused]] uint32_t clc_phase = 0; + int workitem_id = (int)blockIdx.x; + while (true) { + const WorkItem it = decode_workitem(workitem_id, num_heads, num_blocks, packed_mtiles_per_seq, magic0, magic1, magic2, q2k_num); + const int q_start = it.sample * seqlen; + const int mtile[2] = { it.mtile0, it.mtile1 }; + + #pragma unroll + for (int m = 0; m < M_TILES_PER_CTA; ++m) { + mbarrier_wait_parity_suspend(smem_ptr_u32(&full_bar_o_epi[m]), full_o_ph.get_phase()); + + if (elect_one_sync()) { + const int tok0 = q_start + mtile[m] * BLOCK; + + tma_store_4d(&tmap_o, 0, BHSD ? it.sample * num_heads + it.head : it.head, + BHSD ? tok0 - q_start : tok0, 0, + smem_ptr_u32(reinterpret_cast(sO_bufs[m]))); + cp_async_bulk_commit_group(); + } + } + + if (elect_one_sync()) { + cp_async_bulk_wait_group_read<1>(); + mbarrier_arrive(smem_ptr_u32(&empty_bar_o_epi[0])); + cp_async_bulk_wait_group_read<0>(); + mbarrier_arrive(smem_ptr_u32(&empty_bar_o_epi[1])); + } + + full_o_ph.advance(); + if constexpr (USE_CLC) { + ClcTileInfo next = clc_fetch_next_tile<1, 1, ClcRasterOrder::AlongN, 1, true>( + clc_full, clc_empty, clc_response, clc_stage, clc_phase, elect_one_sync()); + clc_fetch_next_tile_advance(clc_stage, clc_phase); + if (!next.valid) break; + workitem_id = next.n_tile; + } else { + workitem_id += gridDim.x; + if (workitem_id >= total_workitems) break; + } + } + } + else if (warp_id == W_SCHED) { + setmaxnreg_dec<48>(); + + if constexpr (USE_CLC) { + int prod_stage = 0; uint32_t prod_phase = 1; + int cons_stage = 0; uint32_t cons_phase = 0; + while (true) { + if (lane == 0) + mbarrier_wait_parity_suspend(smem_ptr_u32(&clc_empty[prod_stage]), prod_phase); + __syncwarp(); + clc_arrive_expect_tx_cta(smem_ptr_u32(&clc_full[prod_stage]), 16); + if (lane == 0) + clc_try_cancel_async(smem_ptr_u32(&clc_response[prod_stage * 4]), + smem_ptr_u32(&clc_full[prod_stage])); + advance_stage_phase(prod_stage, prod_phase); + ClcTileInfo next = clc_fetch_next_tile<1, 1, ClcRasterOrder::AlongN, 1, true>( + clc_full, clc_empty, clc_response, cons_stage, cons_phase, elect_one_sync()); + clc_fetch_next_tile_advance(cons_stage, cons_phase); + if (!next.valid) break; + } + + for (int s = 0; s < CLC_STAGES; ++s) { + if (lane == 0) + mbarrier_wait_parity_suspend(smem_ptr_u32(&clc_empty[prod_stage]), prod_phase); + __syncwarp(); + advance_stage_phase(prod_stage, prod_phase); + } + } + } + else if (warp_id >= W_CORR0 && warp_id < W_MMA) { + setmaxnreg_dec<80>(); + + const int corr_warp_id = warp_id - W_CORR0; + const int corr_tid = corr_warp_id * 32 + lane; + const int corr_row = BLK128 ? corr_tid : (corr_tid & 63); + const bool kv_half0 = BLK128 || corr_tid < 64; + [[maybe_unused]] PhaseTracker<1> alpha_ph; + PhaseTracker<1> o_acc_ph; + PhaseTracker<1> o_epi_empty_ph; + + #pragma unroll + for (int i = 0; i < M_TILES_PER_CTA; ++i) { + mbarrier_arrive(smem_ptr_u32(&empty_bar_spo[i])); + mbarrier_arrive(smem_ptr_u32(&empty_bar_alpha_and_l[i])); + } + + [[maybe_unused]] int clc_stage = 0; + [[maybe_unused]] uint32_t clc_phase = 0; + int workitem_id = (int)blockIdx.x; + while (true) { + const WorkItem it = decode_workitem(workitem_id, num_heads, num_blocks, packed_mtiles_per_seq, magic0, magic1, magic2, q2k_num); + const int num_k_tiles = (it.num_kv_blocks + BLOCKS_PER_KTILE - 1) / BLOCKS_PER_KTILE; + + #pragma unroll + for (int i = 0; i < M_TILES_PER_CTA; ++i) { + if constexpr (FULL_NAMED_BAR) full_bar_wait(i, corr_warp_id); + else mbarrier_wait_parity_suspend(smem_ptr_u32(&full_bar_alpha[i]), alpha_ph.get_phase()); + mbarrier_arrive(smem_ptr_u32(&empty_bar_alpha_and_l[i])); + } + if constexpr (!FULL_NAMED_BAR) alpha_ph.advance(); + + for (int k = 1; k < num_k_tiles; ++k) { + #pragma unroll + for (int i = 0; i < M_TILES_PER_CTA; ++i) { + if constexpr (FULL_NAMED_BAR) full_bar_wait(i, corr_warp_id); + else mbarrier_wait_parity_suspend(smem_ptr_u32(&full_bar_alpha[i]), alpha_ph.get_phase()); + + const float alpha = alpha_and_l_smem[(i * STAT_REGIONS + 0) * STATS + corr_tid]; + if constexpr (!SOFTMAX_THROTTLE) mbarrier_arrive(smem_ptr_u32(&empty_bar_alpha_and_l[i])); + + bool skip = __all_sync(0xffffffffu, alpha == 1.0f); + if (!skip) { + + const uint32_t o_tmem_addr = tmem_base + (uint32_t)(2 * S_COLS + i * O_COLS) + ((uint32_t)(corr_warp_id * 32) << 16); + + const float2 alpha2 = f32x2_splat(alpha); + + #pragma unroll + for (int c0 = 0; c0 < HEAD_DIM; c0 += 16) { + uint32_t o_regs[16]; + tcgen05_ld_32x32b_x16(o_tmem_addr + (uint32_t)c0, o_regs); + float2* o2 = reinterpret_cast(o_regs); + #pragma unroll + for (int e = 0; e < 8; ++e) o2[e] = fmul2(o2[e], alpha2); + tcgen05_st_32x32b_x16(o_tmem_addr + (uint32_t)c0, o_regs); + } + tcgen05_wait_st(); + + tcgen05_fence_before_thread_sync(); + } + if constexpr (SOFTMAX_THROTTLE) mbarrier_arrive(smem_ptr_u32(&empty_bar_alpha_and_l[i])); + mbarrier_arrive(smem_ptr_u32(&empty_bar_spo[i])); + } + if constexpr (!FULL_NAMED_BAR) alpha_ph.advance(); + } + + #pragma unroll + for (int i = 0; i < M_TILES_PER_CTA; ++i) { + mbarrier_wait_parity_suspend(smem_ptr_u32(&full_bar_o_acc[i]), o_acc_ph.get_phase()); + if constexpr (FULL_NAMED_BAR) full_bar_wait(i, corr_warp_id); + else mbarrier_wait_parity_suspend(smem_ptr_u32(&full_bar_l[i]), o_acc_ph.get_phase()); + + float scale_own; + if constexpr (BLK128) { + + const float l = alpha_and_l_smem[(i * STAT_REGIONS + 1) * STATS + corr_tid]; + mbarrier_arrive(smem_ptr_u32(&empty_bar_alpha_and_l[i])); + scale_own = (l > 0.f) ? rcp_approx_ftz_f32(l) : 0.f; + } else { + const float l_own = alpha_and_l_smem[(i * STAT_REGIONS + 1) * STATS + corr_tid]; + const float m_own = alpha_and_l_smem[(i * STAT_REGIONS + 2) * STATS + corr_tid]; + const float l_par = alpha_and_l_smem[(i * STAT_REGIONS + 1) * STATS + (corr_tid ^ 64)]; + const float m_par = alpha_and_l_smem[(i * STAT_REGIONS + 2) * STATS + (corr_tid ^ 64)]; + mbarrier_arrive(smem_ptr_u32(&empty_bar_alpha_and_l[i])); + const float d = (m_own - m_par) * scale_log2; + const float beta_lo = ex2_approx_f32(-fabsf(d)); + const float beta_own = (d >= 0.f) ? 1.f : beta_lo; + const float beta_par = (d >= 0.f) ? beta_lo : 1.f; + const float l_tot = beta_own * l_own + beta_par * l_par; + scale_own = (l_tot > 0.f) ? beta_own * rcp_approx_ftz_f32(l_tot) : 0.f; + + if (lse_out != nullptr && (corr_tid >> 6) == 0) { + const float m_tot = (d >= 0.f) ? m_own : m_par; + const int q_row = (i == 0 ? it.mtile0 : it.mtile1) * BLOCK + (corr_tid & 63); + const float l_safe = (l_tot > 0.f) ? l_tot : 1.0f; + lse_out[((long)it.sample * num_heads + it.head) * (long)seqlen + q_row] = + m_tot * scale_log2 + __log2f(l_safe); + } + } + const float2 scale2 = f32x2_splat(scale_own); + const uint32_t o_tmem_addr = tmem_base + (uint32_t)(2 * S_COLS + i * O_COLS) + ((uint32_t)(corr_warp_id * 32) << 16); + if constexpr (BLK128) { + + #pragma unroll + for (int c0 = 0; c0 < HEAD_DIM; c0 += 16) { + uint32_t o_regs[16]; + tcgen05_ld_32x32b_x16(o_tmem_addr + (uint32_t)c0, o_regs); + if (c0 == 0) mbarrier_wait_parity_suspend(smem_ptr_u32(&empty_bar_o_epi[i]), o_epi_empty_ph.get_phase()); + const float2* o2 = reinterpret_cast(o_regs); + const int s = c0 / SUB_COLS_BF16; + const int v_base = (c0 % SUB_COLS_BF16) / 8; + __nv_bfloat16* so_sub = sO_bufs[i] + s * (M_TILE * SUB_COLS_BF16); + #pragma unroll + for (int vv = 0; vv < 2; ++vv) { + const int v = v_base + vv; + const float2 r0 = fmul2(o2[vv * 4 + 0], scale2); + const float2 r1 = fmul2(o2[vv * 4 + 1], scale2); + const float2 r2 = fmul2(o2[vv * 4 + 2], scale2); + const float2 r3 = fmul2(o2[vv * 4 + 3], scale2); + uint4 packed; + packed.x = cvt_f32x2_to_bf16x2(r0.x, r0.y); + packed.y = cvt_f32x2_to_bf16x2(r1.x, r1.y); + packed.z = cvt_f32x2_to_bf16x2(r2.x, r2.y); + packed.w = cvt_f32x2_to_bf16x2(r3.x, r3.y); + *reinterpret_cast(&so_sub[corr_row * SUB_COLS_BF16 + (v ^ (corr_row & 7)) * 8]) = packed; + } + } + } else { + + mbarrier_wait_parity_suspend(smem_ptr_u32(&empty_bar_o_epi[i]), o_epi_empty_ph.get_phase()); + if (!kv_half0) { + #pragma unroll + for (int c0 = 0; c0 < HEAD_DIM; c0 += 16) { + uint32_t o_regs[16]; + tcgen05_ld_32x32b_x16(o_tmem_addr + (uint32_t)c0, o_regs); + float2* o2 = reinterpret_cast(o_regs); + #pragma unroll + for (int e = 0; e < 8; ++e) o2[e] = fmul2(o2[e], scale2); + const int s = c0 / SUB_COLS_BF16; + const int v_base = (c0 % SUB_COLS_BF16) / 8; + __nv_bfloat16* so_sub = sO_bufs[i] + s * (M_TILE * SUB_COLS_BF16); + #pragma unroll + for (int vv = 0; vv < 2; ++vv) { + const int v = v_base + vv; + uint4 packed; + packed.x = cvt_f32x2_to_bf16x2(o2[vv * 4 + 0].x, o2[vv * 4 + 0].y); + packed.y = cvt_f32x2_to_bf16x2(o2[vv * 4 + 1].x, o2[vv * 4 + 1].y); + packed.z = cvt_f32x2_to_bf16x2(o2[vv * 4 + 2].x, o2[vv * 4 + 2].y); + packed.w = cvt_f32x2_to_bf16x2(o2[vv * 4 + 3].x, o2[vv * 4 + 3].y); + *reinterpret_cast(&so_sub[corr_row * SUB_COLS_BF16 + (v ^ (corr_row & 7)) * 8]) = packed; + } + } + } + bar_sync<9>(128); + if (kv_half0) { + #pragma unroll + for (int c0 = 0; c0 < HEAD_DIM; c0 += 16) { + uint32_t o_regs[16]; + tcgen05_ld_32x32b_x16(o_tmem_addr + (uint32_t)c0, o_regs); + float2* o2 = reinterpret_cast(o_regs); + #pragma unroll + for (int e = 0; e < 8; ++e) o2[e] = fmul2(o2[e], scale2); + const int s = c0 / SUB_COLS_BF16; + const int v_base = (c0 % SUB_COLS_BF16) / 8; + __nv_bfloat16* so_sub = sO_bufs[i] + s * (M_TILE * SUB_COLS_BF16); + #pragma unroll + for (int vv = 0; vv < 2; ++vv) { + const int v = v_base + vv; + __nv_bfloat16* dst = &so_sub[corr_row * SUB_COLS_BF16 + (v ^ (corr_row & 7)) * 8]; + uint4 h1 = *reinterpret_cast(dst); + o2[vv * 4 + 0] = fadd2(o2[vv * 4 + 0], __bfloat1622float2(*reinterpret_cast<__nv_bfloat162*>(&h1.x))); + o2[vv * 4 + 1] = fadd2(o2[vv * 4 + 1], __bfloat1622float2(*reinterpret_cast<__nv_bfloat162*>(&h1.y))); + o2[vv * 4 + 2] = fadd2(o2[vv * 4 + 2], __bfloat1622float2(*reinterpret_cast<__nv_bfloat162*>(&h1.z))); + o2[vv * 4 + 3] = fadd2(o2[vv * 4 + 3], __bfloat1622float2(*reinterpret_cast<__nv_bfloat162*>(&h1.w))); + uint4 packed; + packed.x = cvt_f32x2_to_bf16x2(o2[vv * 4 + 0].x, o2[vv * 4 + 0].y); + packed.y = cvt_f32x2_to_bf16x2(o2[vv * 4 + 1].x, o2[vv * 4 + 1].y); + packed.z = cvt_f32x2_to_bf16x2(o2[vv * 4 + 2].x, o2[vv * 4 + 2].y); + packed.w = cvt_f32x2_to_bf16x2(o2[vv * 4 + 3].x, o2[vv * 4 + 3].y); + *reinterpret_cast(dst) = packed; + } + } + } + bar_sync<9>(128); + } + + tcgen05_fence_before_thread_sync(); + + mbarrier_arrive(smem_ptr_u32(&empty_bar_spo[i])); + + fence_proxy_async_shared(); + + mbarrier_arrive(smem_ptr_u32(&full_bar_o_epi[i])); + } + o_acc_ph.advance(); + o_epi_empty_ph.advance(); + + if constexpr (USE_CLC) { + ClcTileInfo next = clc_fetch_next_tile<1, 1, ClcRasterOrder::AlongN, 1, true>( + clc_full, clc_empty, clc_response, clc_stage, clc_phase, elect_one_sync()); + clc_fetch_next_tile_advance(clc_stage, clc_phase); + if (!next.valid) break; + workitem_id = next.n_tile; + } else { + workitem_id += gridDim.x; + if (workitem_id >= total_workitems) break; + } + } + } + else { + setmaxnreg_inc<192>(); + + const int warp_id_u = __shfl_sync(0xffffffffu, warp_id, 0); + const int m_tile = warp_id_u < 4 ? 0 : 1; + const int warp_in_group = warp_id_u & 3; + + const int sm_tid = warp_in_group * 32 + lane; + const uint32_t alpha_slot_u32 = smem_ptr_u32(&alpha_and_l_smem[(m_tile * STAT_REGIONS + 0) * STATS + sm_tid]); + const uint32_t s_tmem_addr = tmem_base + (uint32_t)(m_tile * S_COLS) + ((uint32_t)(warp_in_group * 32) << 16); + PhaseTracker<1> spo_ph; + PhaseTracker<1> scale_empty_ph; + [[maybe_unused]] int clc_stage = 0; + [[maybe_unused]] uint32_t clc_phase = 0; + + int workitem_id = (int)blockIdx.x; + while (true) { + const WorkItem it = decode_workitem(workitem_id, num_heads, num_blocks, packed_mtiles_per_seq, magic0, magic1, magic2, q2k_num); + const int num_k_tiles = (it.num_kv_blocks + BLOCKS_PER_KTILE - 1) / BLOCKS_PER_KTILE; + + float m_run = -INFINITY, l_run = 0.f; + mbarrier_wait_parity_suspend(smem_ptr_u32(&empty_bar_alpha_and_l[m_tile]), scale_empty_ph.get_phase()); + scale_empty_ph.advance(); + + int thr_window = -(1 << 30); + int thr_cache0 = BLOCK, thr_cache1 = BLOCK; + auto get_vbs_thresholds = [&](int k, int gqb_mt, int half, int& t0, int& t1) { + if (k >= thr_window + 32) { + thr_window = k & ~31; + const int kk = thr_window + lane; + // THIS tile's own count (this warp group serves exactly one m_tile). + // Positions at or past it -- the pair-max padding, and everything in + // an empty row -- get threshold 0, i.e. the whole 64/128-token block + // masks to -inf, regardless of which KV block the load warp fetched. + const int cnt = it.num_kv_blocks_mt[m_tile]; + if constexpr (BLK128) { + thr_cache0 = (kk < cnt) + ? variable_block_sizes[q2k_idx[gqb_mt * max_kv + kk]] : 0; + } else { + const int b0 = kk * BLOCKS_PER_KTILE + 2 * half; + thr_cache0 = (b0 < cnt) + ? variable_block_sizes[q2k_idx[gqb_mt * max_kv + b0]] : 0; + thr_cache1 = (b0 + 1 < cnt) + ? variable_block_sizes[q2k_idx[gqb_mt * max_kv + b0 + 1]] : 0; + } + } + t0 = __shfl_sync(0xffffffffu, thr_cache0, k & 31); + t1 = BLK128 ? 0 : __shfl_sync(0xffffffffu, thr_cache1, k & 31); + }; + auto softmax_step = [&](auto is_first_c, int k) { + constexpr bool IS_FIRST = decltype(is_first_c)::value; + const int gqb_mt_ = (m_tile == 0) ? it.global_mtile0 : it.global_mtile1; + int vbs_thr0_, vbs_thr1_; + get_vbs_thresholds(k, gqb_mt_, warp_in_group >> 1, vbs_thr0_, vbs_thr1_); + mbarrier_wait_parity_suspend(smem_ptr_u32(&full_bar_spo[m_tile]), spo_ph.get_phase()); + + uint32_t s_regs[S_COLS]; + #pragma unroll + for (int c0 = 0; c0 < S_COLS; c0 += S_LD_COLS) { + const uint32_t taddr = s_tmem_addr + (uint32_t)c0; + if constexpr (S_LD_COLS == 32) tcgen05_ld_32x32b_x32 (taddr, *reinterpret_cast(&s_regs[c0])); + else if constexpr (S_LD_COLS == 64) tcgen05_ld_32x32b_x64 (taddr, *reinterpret_cast(&s_regs[c0])); + else if constexpr (S_LD_COLS == 128) tcgen05_ld_32x32b_x128(taddr, *reinterpret_cast(&s_regs[c0])); + } + + float* scores = reinterpret_cast(s_regs); + float2* scores2 = reinterpret_cast(s_regs); + + tcgen05_fence_before_thread_sync(); + + if constexpr (BLK128) { + if (vbs_thr0_ < S_COLS) mask_s_row_r2p(scores, 0, 0, vbs_thr0_); + } else { + if (vbs_thr0_ < 64) mask_s_row_r2p(scores, 0, 0, vbs_thr0_); + if (vbs_thr1_ < 64) mask_s_row_r2p(scores + 64, 0, 0, vbs_thr1_); + } + + float rmax0 = m_run, rmax1 = -INFINITY, rmax2 = -INFINITY, rmax3 = -INFINITY; + #pragma unroll + for (int j = 0; j < S_COLS; j += 8) { + rmax0 = fmaxf(fmaxf(rmax0, scores[j + 0]), scores[j + 1]); + rmax1 = fmaxf(fmaxf(rmax1, scores[j + 2]), scores[j + 3]); + rmax2 = fmaxf(fmaxf(rmax2, scores[j + 4]), scores[j + 5]); + rmax3 = fmaxf(fmaxf(rmax3, scores[j + 6]), scores[j + 7]); + } + float new_m = fmaxf(fmaxf(rmax0, rmax1), fmaxf(rmax2, rmax3)); + + new_m = fmaxf(new_m, -FLT_MAX); + float alpha = 0.0f; + if constexpr (!IS_FIRST) { + + const float acc_scale_ = (m_run - new_m) * scale_log2; + if (acc_scale_ >= -(float)RESCALE_THRESHOLD) { new_m = m_run; alpha = 1.0f; } + else { alpha = ex2_approx_f32(acc_scale_); } + + sts_f32(alpha_slot_u32, alpha); + } + if constexpr (FULL_NAMED_BAR) full_bar_arrive(m_tile, warp_in_group); + else mbarrier_arrive(smem_ptr_u32(&full_bar_alpha[m_tile])); + + const float2 scale2 = f32x2_splat(scale_log2); + const float2 neg_m_scaled2 = f32x2_splat(-new_m * scale_log2); + uint32_t p_regs[S_COLS / 2]; + [[maybe_unused]] float2 lt2_live = make_float2(IS_FIRST ? 0.0f : l_run * alpha, 0.0f); + #pragma unroll + for (int c = 0; c < S_COLS / 2; ++c) { + const float2 a2 = ffma2(scores2[c], scale2, neg_m_scaled2); + + if constexpr (EX2_EMU) { + const int jj = c / EX2_FRG_PAIRS; + const int kk = 2 * (c % EX2_FRG_PAIRS); + const bool use_hw = (kk % EX2_FREQ < EX2_FREQ - EX2_RES) || (jj >= EX2_FRG_CNT - 1); + scores2[c] = use_hw ? make_float2(ex2_approx_f32(a2.x), ex2_approx_f32(a2.y)) + : ex2_emu_f32x2(a2.x, a2.y); + } else { + scores2[c] = make_float2(ex2_approx_f32(a2.x), ex2_approx_f32(a2.y)); + } + if constexpr (!DEFER_ROWSUM) lt2_live = fadd2(lt2_live, scores2[c]); + p_regs[c] = cvt_f32x2_to_bf16x2(scores2[c].x, scores2[c].y); + } + const uint32_t p_tmem_addr = s_tmem_addr; + + if constexpr (SPLIT_P) { + tcgen05_st_32x32b_x32(p_tmem_addr, *reinterpret_cast(&p_regs[0])); + tcgen05_st_32x32b_x16(p_tmem_addr + 32, *reinterpret_cast(&p_regs[32])); + tcgen05_wait_st(); + tcgen05_fence_before_thread_sync(); + mbarrier_arrive(smem_ptr_u32(&empty_bar_spo[m_tile])); + tcgen05_st_32x32b_x16(p_tmem_addr + SPLIT_P_COL, *reinterpret_cast(&p_regs[SPLIT_P_COL])); + tcgen05_wait_st(); + tcgen05_fence_before_thread_sync(); + mbarrier_arrive(smem_ptr_u32(&full_bar_p_last[m_tile])); + } else { + tcgen05_st_32x32b_x32(p_tmem_addr, *reinterpret_cast(&p_regs[0])); + tcgen05_st_32x32b_x32(p_tmem_addr + 32, *reinterpret_cast(&p_regs[32])); + tcgen05_wait_st(); + tcgen05_fence_before_thread_sync(); + mbarrier_arrive(smem_ptr_u32(&empty_bar_spo[m_tile])); + } + + spo_ph.advance(); + float2 lt2 = lt2_live; + if constexpr (DEFER_ROWSUM) { + + float2 lt2a = make_float2(IS_FIRST ? 0.0f : l_run * alpha, 0.0f), lt2b = make_float2(0.f, 0.f); + float2 lt2c = make_float2(0.f, 0.f), lt2d = make_float2(0.f, 0.f); + #pragma unroll + for (int c = 0; c < S_COLS / 2; c += 4) { + lt2a = fadd2(lt2a, scores2[c + 0]); + lt2b = fadd2(lt2b, scores2[c + 1]); + lt2c = fadd2(lt2c, scores2[c + 2]); + lt2d = fadd2(lt2d, scores2[c + 3]); + } + lt2 = fadd2(fadd2(lt2a, lt2b), fadd2(lt2c, lt2d)); + } + l_run = lt2.x + lt2.y; m_run = new_m; + mbarrier_wait_parity_suspend(smem_ptr_u32(&empty_bar_alpha_and_l[m_tile]), scale_empty_ph.get_phase()); + scale_empty_ph.advance(); + }; + if (num_k_tiles > 0) softmax_step(std::true_type{}, 0); + for (int k = 1; k < num_k_tiles; ++k) softmax_step(std::false_type{}, k); + + if constexpr (BLK128) { + if (lse_out != nullptr) { + const int q_row = (m_tile == 0 ? it.mtile0 : it.mtile1) * BLOCK + sm_tid; + const float l_safe = (l_run > 0.f) ? l_run : 1.0f; + lse_out[((long)it.sample * num_heads + it.head) * (long)seqlen + q_row] = + m_run * scale_log2 + __log2f(l_safe); + } + } + + alpha_and_l_smem[(m_tile * STAT_REGIONS + 1) * STATS + sm_tid] = l_run; + if constexpr (!BLK128) + alpha_and_l_smem[(m_tile * STAT_REGIONS + 2) * STATS + sm_tid] = m_run; + if constexpr (FULL_NAMED_BAR) full_bar_arrive(m_tile, warp_in_group); + else mbarrier_arrive(smem_ptr_u32(&full_bar_l[m_tile])); + + if constexpr (USE_CLC) { + ClcTileInfo next = clc_fetch_next_tile<1, 1, ClcRasterOrder::AlongN, 1, true>( + clc_full, clc_empty, clc_response, clc_stage, clc_phase, elect_one_sync()); + clc_fetch_next_tile_advance(clc_stage, clc_phase); + if (!next.valid) break; + workitem_id = next.n_tile; + } else { + workitem_id += gridDim.x; + if (workitem_id >= total_workitems) break; + } + } + } + __syncthreads(); + if (warp_id == 0) tcgen05_dealloc<1>(tmem_base, TMEM_TOTAL); +#endif // host pass or sm_100a device pass (multi-arch guard; see note at the top of the body) +} + +} // namespace VSA_NAMESPACE + +#endif diff --git a/python/sglang/kernels/jit/csrc/diffusion/vsa_block_sparse_sm100/block_sparse_launch_sm100a.cuh b/python/sglang/kernels/jit/csrc/diffusion/vsa_block_sparse_sm100/block_sparse_launch_sm100a.cuh new file mode 100644 index 0000000000..642b19a692 --- /dev/null +++ b/python/sglang/kernels/jit/csrc/diffusion/vsa_block_sparse_sm100/block_sparse_launch_sm100a.cuh @@ -0,0 +1,205 @@ +// Copied and adapted from: https://github.com/hao-ai-lab/FastVideo +// (fastvideo-kernel/csrc/attention/block_sparse_launch_sm100a.cuh, Apache-2.0). Inference-only +// forward; the sm_103a device pass is admitted alongside sm_100a. + +#ifndef BLOCK_SPARSE_VSA_LAUNCH_SM100A_CUH +#define BLOCK_SPARSE_VSA_LAUNCH_SM100A_CUH + +// Launch surface for the sm_100a VSA block-sparse FMHA forward. +// +// Everything a caller needs: a POD argument struct, a predicate saying whether this build can +// run those arguments, and one launch entry point. The benchmark in +// block_sparse_bench_sm100a.cu and the torch binding both go through here, so there is +// one tensormap construction and one launch configuration rather than two that can drift. +// +// Two compile-time knobs select the four builds: +// VSA_BLK128 false -> 64-token sparse blocks, true -> 128-token +// VSA_BHSD false -> [token][head][dim] (BSHD), true -> [batch][head][token][dim] (BHSD) + +#include "block_sparse_kernel_sm100a.cuh" + +namespace VSA_NAMESPACE { + +struct BlockSparseVsaArgs { + const __nv_bfloat16* q; + const __nv_bfloat16* k; + const __nv_bfloat16* v; // natural layout; only blk128 reads it (blk64 still needs v_t) + const __nv_bfloat16* v_t; // unused: kept so the bench's V_T buffer still binds + __nv_bfloat16* o; + float* lse; // [batch, num_heads, seqlen] fp32, or nullptr + + const int* q2k_idx; // [batch*num_heads*num_blocks, max_kv] int32 + const int* q2k_num; // [batch*num_heads*num_blocks] int32 + const int* variable_block_sizes; // [num_blocks] int32, valid tokens per block + + int batch; + int num_heads; + int seqlen; + int head_dim; + int num_blocks; + int max_kv; + float sm_scale; +}; + +// cudaSuccess iff this build can run `a`. Deliberately conservative: the caller is expected +// to fall back to its own implementation rather than get a wrong answer. +__host__ inline cudaError_t block_sparse_supported(const BlockSparseVsaArgs& a) { + if (a.head_dim != HEAD_DIM) return cudaErrorInvalidValue; // compile-time in the kernel + if (a.num_blocks % 2 != 0) return cudaErrorInvalidValue; // a CTA owns an adjacent pair + if (a.seqlen != a.num_blocks * BLOCK) return cudaErrorInvalidValue; + if (a.max_kv < 1 || a.num_blocks < 1) return cudaErrorInvalidValue; + if (a.q == nullptr || a.k == nullptr || a.o == nullptr) return cudaErrorInvalidValue; + if (a.q2k_idx == nullptr || a.q2k_num == nullptr) return cudaErrorInvalidValue; + // FastVideo always supplies this; without it padded keys would be attended as real zeros. + if (a.variable_block_sizes == nullptr) return cudaErrorInvalidValue; + // V is read MN-major at BOTH block sizes now, so no pre-transposed V_T is ever needed. + if (a.v == nullptr) return cudaErrorInvalidValue; + return cudaSuccess; +} + +__host__ inline cudaError_t launch_block_sparse_sm100a(const BlockSparseVsaArgs& a, + cudaStream_t stream) { + const cudaError_t sup = block_sparse_supported(a); + if (sup != cudaSuccess) return sup; + + const int B = a.batch, H = a.num_heads, S = a.seqlen, hd = a.head_dim; + const int num_blocks = a.num_blocks, max_kv = a.max_kv; + const long tq = (long)B * S; + const int packed_mtiles_per_seq = num_blocks / 2; + const int total_work = B * H * packed_mtiles_per_seq; + constexpr bool BHSD = VSA_BHSD; + + CUtensorMap tq_, tk_, tvt_, tv_, to_; + { + uint64_t gd[4] = { (uint64_t)SUB_COLS_BF16, BHSD ? (uint64_t)((long)B * H) : (uint64_t)H, + BHSD ? (uint64_t)S : (uint64_t)tq, (uint64_t)Q_SUBTILES }; + uint64_t gs[3] = { BHSD ? (uint64_t)((long)S * hd) * 2u : (uint64_t)hd * 2u, + BHSD ? (uint64_t)hd * 2u : (uint64_t)((long)H * hd) * 2u, + (uint64_t)SUB_COLS_BF16 * 2u }; + uint32_t bd[4] = { (uint32_t)SUB_COLS_BF16, 1u, (uint32_t)M_TILE, (uint32_t)Q_SUBTILES }; + uint32_t es[4] = { 1u, 1u, 1u, 1u }; + if (cuTensorMapEncodeTiled(&tq_, CU_TENSOR_MAP_DATA_TYPE_BFLOAT16, 4, + const_cast<__nv_bfloat16*>(a.q), gd, gs, bd, es, + CU_TENSOR_MAP_INTERLEAVE_NONE, CU_TENSOR_MAP_SWIZZLE_128B, + CU_TENSOR_MAP_L2_PROMOTION_L2_128B, + CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE) != CUDA_SUCCESS) + return cudaErrorInvalidValue; + if (cuTensorMapEncodeTiled(&to_, CU_TENSOR_MAP_DATA_TYPE_BFLOAT16, 4, a.o, gd, gs, bd, es, + CU_TENSOR_MAP_INTERLEAVE_NONE, CU_TENSOR_MAP_SWIZZLE_128B, + CU_TENSOR_MAP_L2_PROMOTION_L2_128B, + CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE) != CUDA_SUCCESS) + return cudaErrorInvalidValue; + } + { + uint64_t gd[4] = { (uint64_t)SUB_COLS_BF16, + BHSD ? (uint64_t)S : (uint64_t)tq, + BHSD ? (uint64_t)(hd / SUB_COLS_BF16) + : (uint64_t)((long)H * hd / SUB_COLS_BF16), + (uint64_t)((long)B * H) }; + uint64_t gs[3] = { BHSD ? (uint64_t)hd * 2u : (uint64_t)((long)H * hd) * 2u, + (uint64_t)SUB_COLS_BF16 * 2u, + (uint64_t)((long)S * hd) * 2u }; + uint32_t bd[4] = { (uint32_t)SUB_COLS_BF16, (uint32_t)BLOCK, + BLK128 ? (uint32_t)K_SUBTILES : 1u, 1u }; + uint32_t es[4] = { 1u, 1u, 1u, 1u }; + if (cuTensorMapEncodeTiled(&tk_, CU_TENSOR_MAP_DATA_TYPE_BFLOAT16, BHSD ? 4 : 3, + const_cast<__nv_bfloat16*>(a.k), gd, gs, bd, es, + CU_TENSOR_MAP_INTERLEAVE_NONE, CU_TENSOR_MAP_SWIZZLE_128B, + CU_TENSOR_MAP_L2_PROMOTION_L2_128B, + CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE) != CUDA_SUCCESS) + return cudaErrorInvalidValue; + // V map is byte-for-byte the K map over a.v: MN-major V needs no transpose (blk128). + const __nv_bfloat16* vbase = a.v ? a.v : a.k; + if (cuTensorMapEncodeTiled(&tv_, CU_TENSOR_MAP_DATA_TYPE_BFLOAT16, BHSD ? 4 : 3, + const_cast<__nv_bfloat16*>(vbase), gd, gs, bd, es, + CU_TENSOR_MAP_INTERLEAVE_NONE, CU_TENSOR_MAP_SWIZZLE_128B, + CU_TENSOR_MAP_L2_PROMOTION_L2_128B, + CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE) != CUDA_SUCCESS) + return cudaErrorInvalidValue; + } + // V_T map: blk64 only. Unused at blk128 but must still be a valid tensormap to pass by value. + { + const __nv_bfloat16* vt = a.v_t ? a.v_t : a.k; + if constexpr (BLK128) { + uint64_t gd[3] = { (uint64_t)SUB_COLS_BF16, (uint64_t)((long)H * hd), + (uint64_t)((long)tq / SUB_COLS_BF16) }; + uint64_t gs[2] = { (uint64_t)tq * 2u, (uint64_t)SUB_COLS_BF16 * 2u }; + uint32_t bd[3] = { (uint32_t)SUB_COLS_BF16, (uint32_t)hd, (uint32_t)V_SUBTILES }; + uint32_t es[3] = { 1u, 1u, 1u }; + if (cuTensorMapEncodeTiled(&tvt_, CU_TENSOR_MAP_DATA_TYPE_BFLOAT16, 3, + const_cast<__nv_bfloat16*>(vt), gd, gs, bd, es, + CU_TENSOR_MAP_INTERLEAVE_NONE, CU_TENSOR_MAP_SWIZZLE_128B, + CU_TENSOR_MAP_L2_PROMOTION_L2_128B, + CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE) != CUDA_SUCCESS) + return cudaErrorInvalidValue; + } else { + if (make_tma_2d_tiled(&tvt_, const_cast<__nv_bfloat16*>(vt), (long)H * hd, (int)tq, hd, + SUB_COLS_BF16, 2, CU_TENSOR_MAP_DATA_TYPE_BFLOAT16, + CU_TENSOR_MAP_SWIZZLE_128B) != cudaSuccess) + return cudaErrorInvalidValue; + } + } + + const size_t smem = + (size_t)2 * Q_TILE_BYTES + NUM_KV_STAGES * KV_RING_SLOT_BYTES + + (size_t)2 * M_TILE * HEAD_DIM * sizeof(__nv_bfloat16) + + (2 * NUM_KV_STAGES + 22) * 8 + + (size_t)CLC_STAGES * (2 * 8 + 16) + 16 + + 8 + + (size_t)2 * STAT_REGIONS * STATS * sizeof(float) + + 256; + +#ifndef VSA_NAMED_BAR +#define VSA_NAMED_BAR false +#endif +#ifndef VSA_THROTTLE +#define VSA_THROTTLE false +#endif +#ifndef VSA_USE_CLC +#define VSA_USE_CLC true +#endif + constexpr bool FULL_NAMED_BAR = VSA_NAMED_BAR, EX2_EMU = true, SPLIT_P = true, + SOFTMAX_THROTTLE = VSA_THROTTLE, USE_CLC = VSA_USE_CLC, + Q_RASTER = true, MHA = true; + auto kfn = &fmha_context_bf16_gen_kernel<32, FULL_NAMED_BAR, EX2_EMU, SPLIT_P, + SOFTMAX_THROTTLE, USE_CLC, Q_RASTER, MHA, + /*RESCALE_THRESHOLD=*/8, /*BHSD=*/VSA_BHSD>; + cudaError_t e = cudaFuncSetAttribute(kfn, cudaFuncAttributeMaxDynamicSharedMemorySize, (int)smem); + if (e != cudaSuccess) return e; + + const unsigned long long magic0 = make_magic((unsigned)(H * packed_mtiles_per_seq)); + const unsigned long long magic1 = make_magic((unsigned)H); + const unsigned long long magic2 = make_magic((unsigned)packed_mtiles_per_seq); + const float scale_log2 = a.sm_scale * (float)M_LOG2E; + + int numSM = 0; + e = cudaDeviceGetAttribute(&numSM, cudaDevAttrMultiProcessorCount, 0); + if (e != cudaSuccess) return e; + const int num_ctas = USE_CLC ? total_work : (total_work < numSM ? total_work : numSM); + dim3 grid(num_ctas, 1, 1), block(N_WARPS * 32, 1, 1); + + if (USE_CLC) { + cudaLaunchConfig_t cfg = {}; + cfg.gridDim = grid; cfg.blockDim = block; cfg.dynamicSmemBytes = smem; cfg.stream = stream; + cudaLaunchAttribute cfgAttr[1]; + cfgAttr[0].id = cudaLaunchAttributeClusterDimension; + cfgAttr[0].val.clusterDim.x = 1; cfgAttr[0].val.clusterDim.y = 1; + cfgAttr[0].val.clusterDim.z = 1; + cfg.attrs = cfgAttr; cfg.numAttrs = 1; + return cudaLaunchKernelEx(&cfg, kfn, tq_, tk_, tvt_, tv_, to_, S, H, scale_log2, B, + num_blocks, packed_mtiles_per_seq, max_kv, magic0, magic1, magic2, + a.q2k_idx, a.q2k_num, a.variable_block_sizes, a.lse); + } + kfn<<>>(tq_, tk_, tvt_, tv_, to_, S, H, scale_log2, B, num_blocks, + packed_mtiles_per_seq, max_kv, magic0, magic1, magic2, + a.q2k_idx, a.q2k_num, a.variable_block_sizes, a.lse); + return cudaGetLastError(); +} + +} // namespace VSA_NAMESPACE + +// Callers (the bench, the torch binding) keep using unqualified names; each translation unit +// only ever sees the one configuration its VSA_BLK128 selected. +using namespace VSA_NAMESPACE; + +#endif // BLOCK_SPARSE_VSA_LAUNCH_SM100A_CUH diff --git a/python/sglang/kernels/jit/csrc/diffusion/vsa_block_sparse_sm100/primitives.cuh b/python/sglang/kernels/jit/csrc/diffusion/vsa_block_sparse_sm100/primitives.cuh new file mode 100644 index 0000000000..9fb68c6826 --- /dev/null +++ b/python/sglang/kernels/jit/csrc/diffusion/vsa_block_sparse_sm100/primitives.cuh @@ -0,0 +1,880 @@ +// Copied and adapted from: https://github.com/hao-ai-lab/FastVideo +// (fastvideo-kernel/csrc/attention/primitives.cuh, Apache-2.0). Inference-only +// forward; the sm_103a device pass is admitted alongside sm_100a. + +// primitives.cuh -- device primitives for the sm_100a VSA block-sparse attention +// forward: tcgen05 (alloc / mma / ld / st / commit / wait / fence), TMA load / store / +// tensormap, mbarrier, cluster launch control, setmaxnreg, fast math, and the FMHA helpers. +// +// Generated and pruned to what the kernel reaches -- do not edit by hand. +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef CUDA_CHECK +#define CUDA_CHECK(stmt) do { \ + cudaError_t _e = (stmt); \ + if (_e != cudaSuccess) { \ + fprintf(stderr, "CUDA error %s:%d: %s -> %s\n", \ + __FILE__, __LINE__, #stmt, cudaGetErrorString(_e)); \ + std::exit(1); \ + } \ + } while (0) +#endif + +__device__ __forceinline__ +uint64_t mbarrier_arrive(uint32_t mbar_smem) { + uint64_t state; + asm volatile("mbarrier.arrive.shared::cta.b64 %0, [%1];\n" + : "=l"(state) : "r"(mbar_smem) : "memory"); + return state; +} + +__device__ __forceinline__ +void mbarrier_arrive_nostate(uint32_t mbar_smem) { + asm volatile("mbarrier.arrive.shared::cta.b64 _, [%0];\n" + :: "r"(mbar_smem) : "memory"); +} + +__device__ __forceinline__ +void mbarrier_arrive_cluster_default(uint32_t cluster_smem_addr) { + asm volatile("mbarrier.arrive.shared::cluster.b64 _, [%0];\n" + :: "r"(cluster_smem_addr) : "memory"); +} + +__device__ __forceinline__ +void mbarrier_arrive_expect_tx(uint32_t mbar_smem, uint32_t expected_bytes) { + asm volatile("mbarrier.arrive.expect_tx.release.cta.shared::cta.b64 _, [%0], %1;\n" + :: "r"(mbar_smem), "r"(expected_bytes) : "memory"); +} + +__device__ __forceinline__ +void mbarrier_wait_parity_suspend(uint32_t mbar_smem, uint32_t phase_parity) { + asm volatile( + "{\n" + ".reg .pred P1;\n" + "LAB_WAIT:\n" + "mbarrier.try_wait.parity.shared::cta.b64 P1, [%0], %1, 10000000;\n" + "@!P1 bra.uni LAB_WAIT;\n" + "}\n" + :: "r"(mbar_smem), "r"(phase_parity) : "memory"); +} + +__device__ __forceinline__ +void mbarrier_wait_parity(uint32_t mbar_smem, uint32_t phase_parity) { + asm volatile( + "{\n" + ".reg .pred P1;\n" + "LAB_WAIT_HOT:\n" + "mbarrier.try_wait.parity.shared::cta.b64 P1, [%0], %1;\n" + "@!P1 bra.uni LAB_WAIT_HOT;\n" + "}\n" + :: "r"(mbar_smem), "r"(phase_parity) : "memory"); +} + +__device__ __forceinline__ +void fence_proxy_async_shared_cta() { + asm volatile("fence.proxy.async.shared::cta;\n" ::: "memory"); +} + +__device__ __forceinline__ +void fence_proxy_async_shared() { + asm volatile("fence.proxy.async.shared::cta;\n" ::: "memory"); +} + +__device__ __forceinline__ void clc_try_cancel_async( + uint32_t smem_dst, uint32_t mbar_smem) { + asm volatile( + "clusterlaunchcontrol.try_cancel.async.shared::cta.mbarrier::complete_tx::bytes.b128" + " [%0], [%1];\n" + :: "r"(smem_dst), "r"(mbar_smem) : "memory"); +} + +__device__ __forceinline__ void clc_load_response( + uint32_t smem_slot, uint32_t& r0, uint32_t& r1, + uint32_t& r2, uint32_t& r3) { + asm volatile("ld.shared::cta.v4.b32 {%0, %1, %2, %3}, [%4];\n" + : "=r"(r0), "=r"(r1), "=r"(r2), "=r"(r3) + : "r"(smem_slot)); +} + +template +struct MbarrierPhaseTracker { + + uint32_t phase[NUM_STAGES]; + int idx; + + __device__ __forceinline__ + void init() { + for (int i = 0; i < NUM_STAGES; ++i) phase[i] = 0; + idx = 0; + } + + __device__ __forceinline__ + uint32_t current_phase() const { return phase[idx]; } + + __device__ __forceinline__ + void advance() { + phase[idx] ^= 1u; + idx = (idx + 1) % NUM_STAGES; + } + + __device__ __forceinline__ + int stage() const { return idx; } +}; + +template +struct PhaseTracker { + int stage; + uint32_t phase; + + __device__ __forceinline__ + PhaseTracker() : stage(0), phase(0) {} + + __device__ __forceinline__ + void advance() { + stage++; + if (stage == NUM_STAGES) { + stage = 0; + phase ^= 1; + } + } + + __device__ __forceinline__ + int get_stage() const { return stage; } + + __device__ __forceinline__ + uint32_t get_phase() const { return phase; } +}; + +template +struct EmptyPhaseTracker { + int stage; + uint32_t phase; + + __device__ __forceinline__ + EmptyPhaseTracker() : stage(0), phase(1) {} + + __device__ __forceinline__ + void advance() { + stage++; + if (stage == NUM_STAGES) { + stage = 0; + phase ^= 1; + } + } + + __device__ __forceinline__ + int get_stage() const { return stage; } + + __device__ __forceinline__ + uint32_t get_phase() const { return phase; } +}; + +template +__device__ __forceinline__ +void advance_stage_phase(int& stage, uint32_t& phase) { + ++stage; + if (stage == STAGES) { + stage = 0; + phase ^= 1u; + } +} + +static constexpr uint32_t SM100_CLC_PEER_MASK = 0xFEFFFFFF; + +struct ClcTileInfo { + int m_tile; + int n_tile; + bool valid; +}; + +enum class ClcRasterOrder { AlongN, AlongM }; + +__device__ __forceinline__ +void clc_arrive_expect_tx_cta(uint32_t clc_full_local_addr, uint32_t tx_bytes) { + if ((threadIdx.x & 31) == 0) { + mbarrier_arrive_expect_tx(clc_full_local_addr, tx_bytes); + } +} + +__device__ __forceinline__ +void clc_consumer_release(uint32_t clc_empty_local_addr) { + uint32_t peer0_addr = clc_empty_local_addr & SM100_CLC_PEER_MASK; + mbarrier_arrive_cluster_default(peer0_addr); +} + +__device__ __forceinline__ +void clc_consumer_release_cta(uint32_t clc_empty_local_addr) { + mbarrier_arrive_nostate(clc_empty_local_addr); +} + +template +__device__ __forceinline__ +ClcTileInfo clc_parse_response(uint32_t resp_smem_addr) { + uint32_t d0, d1, d2, d3; + fence_proxy_async_shared_cta(); + clc_load_response(resp_smem_addr, d0, d1, d2, d3); + const int ctaid_x = static_cast(d0); + const int ctaid_y = static_cast(d1 & 0xFFFFu); + const bool valid = (d2 & 1u) != 0u; + (void)d3; + + ClcTileInfo info; + info.valid = valid; + if constexpr (ORDER == ClcRasterOrder::AlongN) { + info.m_tile = ctaid_y / CLUSTER_SHAPE_M; + info.n_tile = ctaid_x / CLUSTER_SHAPE_N; + } else { + info.m_tile = ctaid_x / CLUSTER_SHAPE_M; + info.n_tile = ctaid_y / CLUSTER_SHAPE_N; + } + return info; +} + +template +__device__ __forceinline__ +ClcTileInfo clc_fetch_next_tile( + uint64_t* clc_full_bar, uint64_t* clc_empty_bar, uint32_t* clc_response, + int clc_cons_stage, uint32_t clc_cons_phase, bool do_release) { + uint32_t full_addr = static_cast( + __cvta_generic_to_shared(&clc_full_bar[clc_cons_stage])); + if constexpr (SUSPEND) mbarrier_wait_parity_suspend(full_addr, clc_cons_phase); + else mbarrier_wait_parity(full_addr, clc_cons_phase); + uint32_t resp_addr = static_cast( + __cvta_generic_to_shared(&clc_response[clc_cons_stage * 4])); + ClcTileInfo t = clc_parse_response< + CLUSTER_SHAPE_M, CLUSTER_SHAPE_N, ORDER>(resp_addr); + if (do_release) { + uint32_t empty_local = static_cast( + __cvta_generic_to_shared(&clc_empty_bar[clc_cons_stage])); + if constexpr (CTA_GROUP == 1) { + clc_consumer_release_cta(empty_local); + } else { + clc_consumer_release(empty_local); + } + } + return t; +} + +template +__device__ __forceinline__ +void clc_fetch_next_tile_advance(int& clc_cons_stage, + uint32_t& clc_cons_phase) { + advance_stage_phase(clc_cons_stage, clc_cons_phase); +} + +__device__ __forceinline__ unsigned fdiv(unsigned n, unsigned long long pk) { + unsigned M = (unsigned)pk; + if (M == 0u) return n; + return __umulhi(n, M) >> (unsigned)(pk >> 32); +} +__host__ inline unsigned long long make_magic(unsigned d) { + if (d <= 1u) return 0ULL; + unsigned l = 0; while ((1u << (l + 1)) <= d) ++l; + unsigned p = 31u + l; + unsigned long long m = ((1ull << p) + (unsigned long long)d - 1ull) / d; + return (m & 0xffffffffULL) | ((unsigned long long)(p - 32u) << 32); +} + +template +__device__ __forceinline__ +void bar_sync(uint32_t thread_count) { + static_assert(BARRIER_ID >= 0 && BARRIER_ID <= 15, + "bar.sync: BARRIER_ID must be in [0, 15]"); + asm volatile("bar.sync %0, %1;\n" + :: "n"(BARRIER_ID), "r"(thread_count) : "memory"); +} + +template +__device__ __forceinline__ +void bar_arrive(uint32_t thread_count) { + static_assert(BARRIER_ID >= 0 && BARRIER_ID <= 15, + "bar.arrive: BARRIER_ID must be in [0, 15]"); + asm volatile("bar.arrive %0, %1;\n" + :: "n"(BARRIER_ID), "r"(thread_count) : "memory"); +} + +__device__ __forceinline__ void full_bar_arrive(int m_tile, int band) { + switch (1 + m_tile * 4 + band) { + case 1: bar_arrive<1>(64); break; + case 2: bar_arrive<2>(64); break; + case 3: bar_arrive<3>(64); break; + case 4: bar_arrive<4>(64); break; + case 5: bar_arrive<5>(64); break; + case 6: bar_arrive<6>(64); break; + case 7: bar_arrive<7>(64); break; + case 8: bar_arrive<8>(64); break; + } +} +__device__ __forceinline__ void full_bar_wait(int m_tile, int band) { + switch (1 + m_tile * 4 + band) { + case 1: bar_sync<1>(64); break; + case 2: bar_sync<2>(64); break; + case 3: bar_sync<3>(64); break; + case 4: bar_sync<4>(64); break; + case 5: bar_sync<5>(64); break; + case 6: bar_sync<6>(64); break; + case 7: bar_sync<7>(64); break; + case 8: bar_sync<8>(64); break; + } +} + +template +__device__ __forceinline__ void mask_s_row_r2p(float* scores, int k_offset, int q_pos, int seqlen_k) { + int n_keep = seqlen_k - k_offset; + if constexpr (IS_CAUSAL) { + const int causal = q_pos - k_offset + 1; + n_keep = n_keep < causal ? n_keep : causal; + } + #pragma unroll + for (int s = 0; s < K_TILE / 32; ++s) { + int m = (s + 1) * 32 - n_keep; + m = m < 0 ? 0 : (m > 32 ? 32 : m); + const uint32_t keep = (m >= 32) ? 0u : (0xFFFFFFFFu >> m); + #pragma unroll + for (int i = 0; i < 32; ++i) + if (!(keep & (1u << i))) scores[s * 32 + i] = -INFINITY; + } +} + +template +__device__ __forceinline__ void tcgen05_alloc(uint32_t smem_dst_ptr, + uint32_t n_cols) { + static_assert(CTA_GROUP == 1 || CTA_GROUP == 2, + "tcgen05_alloc: CTA_GROUP must be 1 or 2"); + if constexpr (CTA_GROUP == 1) { + asm volatile( + "tcgen05.alloc.cta_group::1.sync.aligned.shared::cta.b32 [%0], %1;\n" + :: "r"(smem_dst_ptr), "r"(n_cols)); + } else { + asm volatile( + "tcgen05.alloc.cta_group::2.sync.aligned.shared::cta.b32 [%0], %1;\n" + :: "r"(smem_dst_ptr), "r"(n_cols)); + } +} + +__device__ __forceinline__ void tcgen05_st_32x32b_x16( + uint32_t tmem_addr, const uint32_t (&r)[16]) { + asm volatile("tcgen05.st.sync.aligned.32x32b.x16.b32 " + "[%0], {%1,%2,%3,%4,%5,%6,%7,%8,%9,%10," + "%11,%12,%13,%14,%15,%16};\n" + :: "r"(tmem_addr), + "r"(r[0]),"r"(r[1]),"r"(r[2]),"r"(r[3]), + "r"(r[4]),"r"(r[5]),"r"(r[6]),"r"(r[7]), + "r"(r[8]),"r"(r[9]),"r"(r[10]),"r"(r[11]), + "r"(r[12]),"r"(r[13]),"r"(r[14]),"r"(r[15])); +} + +__device__ __forceinline__ void tcgen05_st_32x32b_x32( + uint32_t tmem_addr, const uint32_t (&r)[32]) { + asm volatile("tcgen05.st.sync.aligned.32x32b.x32.b32 " + "[%0], {%1,%2,%3,%4,%5,%6,%7,%8,%9,%10," + "%11,%12,%13,%14,%15,%16,%17,%18,%19,%20," + "%21,%22,%23,%24,%25,%26,%27,%28,%29,%30," + "%31,%32};\n" + :: "r"(tmem_addr), + "r"(r[0]),"r"(r[1]),"r"(r[2]),"r"(r[3]), + "r"(r[4]),"r"(r[5]),"r"(r[6]),"r"(r[7]), + "r"(r[8]),"r"(r[9]),"r"(r[10]),"r"(r[11]), + "r"(r[12]),"r"(r[13]),"r"(r[14]),"r"(r[15]), + "r"(r[16]),"r"(r[17]),"r"(r[18]),"r"(r[19]), + "r"(r[20]),"r"(r[21]),"r"(r[22]),"r"(r[23]), + "r"(r[24]),"r"(r[25]),"r"(r[26]),"r"(r[27]), + "r"(r[28]),"r"(r[29]),"r"(r[30]),"r"(r[31])); +} + +__device__ __forceinline__ void tcgen05_commit1_lead(uint32_t lead, uint32_t mbar_smem_addr) { + asm volatile( + "{\n\t" + ".reg .pred q;\n\t" + "setp.ne.b32 q, %0, 0;\n\t" + "@q tcgen05.commit.cta_group::1.mbarrier::arrive::one.b64 [%1];\n\t" + "}\n" + :: "r"(lead), "r"(mbar_smem_addr)); +} + +__device__ __forceinline__ void tcgen05_wait_st() { + asm volatile("tcgen05.wait::st.sync.aligned;\n" ::: "memory"); +} + +__device__ __forceinline__ void tcgen05_fence_before_thread_sync() { + asm volatile("tcgen05.fence::before_thread_sync;\n" ::: "memory"); +} + +__device__ __forceinline__ +void tma_load_2d(uint32_t smem_dst, const void* tensormap_ptr, + uint32_t mbar_smem, int coord_x, int coord_y) { + asm volatile( + "cp.async.bulk.tensor.2d.shared::cluster.global.tile" + ".mbarrier::complete_tx::bytes" + " [%0], [%1, {%3, %4}], [%2];\n" + :: "r"(smem_dst), "l"(tensormap_ptr), + "r"(mbar_smem), "r"(coord_x), "r"(coord_y) + : "memory"); +} + +__device__ __forceinline__ +void tma_load_3d(uint32_t smem_dst, const void* tensormap_ptr, + uint32_t mbar_smem, int c0, int c1, int c2) { + asm volatile( + "cp.async.bulk.tensor.3d.shared::cluster.global.tile" + ".mbarrier::complete_tx::bytes" + " [%0], [%1, {%3, %4, %5}], [%2];\n" + :: "r"(smem_dst), "l"(tensormap_ptr), + "r"(mbar_smem), "r"(c0), "r"(c1), "r"(c2) + : "memory"); +} + +__device__ __forceinline__ +void tma_load_4d(uint32_t smem_dst, const void* tensormap_ptr, + uint32_t mbar_smem, int c0, int c1, int c2, int c3) { + asm volatile( + "cp.async.bulk.tensor.4d.shared::cluster.global.tile" + ".mbarrier::complete_tx::bytes" + " [%0], [%1, {%3, %4, %5, %6}], [%2];\n" + :: "r"(smem_dst), "l"(tensormap_ptr), + "r"(mbar_smem), "r"(c0), "r"(c1), "r"(c2), "r"(c3) + : "memory"); +} + +template +__device__ __forceinline__ void tcgen05_dealloc(uint32_t tmem_addr, + uint32_t n_cols) { + static_assert(CTA_GROUP == 1 || CTA_GROUP == 2, + "tcgen05_dealloc: CTA_GROUP must be 1 or 2"); + if constexpr (CTA_GROUP == 1) { + asm volatile( + "tcgen05.dealloc.cta_group::1.sync.aligned.b32 %0, %1;\n" + :: "r"(tmem_addr), "r"(n_cols)); + } else { + asm volatile( + "tcgen05.dealloc.cta_group::2.sync.aligned.b32 %0, %1;\n" + :: "r"(tmem_addr), "r"(n_cols)); + } +} + +__device__ __forceinline__ +void tma_store_2d(const void* tensormap_ptr, int coord_x, int coord_y, + uint32_t smem_src) { + asm volatile( + "cp.async.bulk.tensor.2d.global.shared::cta.tile.bulk_group" + " [%0, {%1, %2}], [%3];\n" + :: "l"(tensormap_ptr), "r"(coord_x), "r"(coord_y), + "r"(smem_src) + : "memory"); +} + +__device__ __forceinline__ +void tma_store_3d(const void* tensormap_ptr, int c0, int c1, int c2, + uint32_t smem_src) { + asm volatile( + "cp.async.bulk.tensor.3d.global.shared::cta.tile.bulk_group" + " [%0, {%1, %2, %3}], [%4];\n" + :: "l"(tensormap_ptr), "r"(c0), "r"(c1), "r"(c2), "r"(smem_src) + : "memory"); +} + +__device__ __forceinline__ +void tma_store_4d(const void* tensormap_ptr, int c0, int c1, int c2, int c3, + uint32_t smem_src) { + asm volatile( + "cp.async.bulk.tensor.4d.global.shared::cta.tile.bulk_group" + " [%0, {%1, %2, %3, %4}], [%5];\n" + :: "l"(tensormap_ptr), "r"(c0), "r"(c1), "r"(c2), "r"(c3), "r"(smem_src) + : "memory"); +} + +inline cudaError_t make_tma_2d_tiled( + CUtensorMap* out, + const void* ptr, int rows, int cols, int box_rows, int box_cols, + int elem_bytes, CUtensorMapDataType dtype, + CUtensorMapSwizzle swizzle = CU_TENSOR_MAP_SWIZZLE_128B, + CUtensorMapL2promotion l2 = CU_TENSOR_MAP_L2_PROMOTION_L2_128B, + CUtensorMapFloatOOBfill oob = CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE) { + uint64_t globalDim[2] = { (uint64_t)cols, (uint64_t)rows }; + uint64_t globalStrides[1] = { (uint64_t)cols * (uint64_t)elem_bytes }; + uint32_t boxDim[2] = { (uint32_t)box_cols, (uint32_t)box_rows }; + uint32_t elemStrides[2] = { 1u, 1u }; + + CUresult r = cuTensorMapEncodeTiled( + out, dtype, 2, + const_cast(ptr), globalDim, globalStrides, + boxDim, elemStrides, + CU_TENSOR_MAP_INTERLEAVE_NONE, + swizzle, l2, oob); + return (r == CUDA_SUCCESS) ? cudaSuccess : cudaErrorInvalidValue; +} + +__device__ __forceinline__ +void cp_async_bulk_commit_group() { + asm volatile("cp.async.bulk.commit_group;\n" ::: "memory"); +} + +template +__device__ __forceinline__ +void cp_async_bulk_wait_group_read() { + asm volatile("cp.async.bulk.wait_group.read %0;\n" :: "n"(N) : "memory"); +} + +__device__ __forceinline__ +void mbarrier_init(uint32_t mbar_smem, uint32_t arrive_count) { + asm volatile("mbarrier.init.shared::cta.b64 [%0], %1;\n" + :: "r"(mbar_smem), "r"(arrive_count) : "memory"); +} + +template +__device__ __forceinline__ void tcgen05_relinquish_alloc_permit() { + static_assert(CTA_GROUP == 1 || CTA_GROUP == 2, + "tcgen05_relinquish_alloc_permit: CTA_GROUP must be 1 or 2"); + if constexpr (CTA_GROUP == 1) { + asm volatile("tcgen05.relinquish_alloc_permit.cta_group::1.sync.aligned;\n" ::); + } else { + asm volatile("tcgen05.relinquish_alloc_permit.cta_group::2.sync.aligned;\n" ::); + } +} + +__device__ __forceinline__ +void fence_mbarrier_init_release_cluster() { + asm volatile("fence.mbarrier_init.release.cluster;\n" ::: "memory"); +} + +__device__ __forceinline__ void tcgen05_mma_f16_ss_lead(uint32_t lead, + uint32_t tmem_c, uint64_t desc_a, uint64_t desc_b, uint32_t idesc, + bool enable_input_d) { + asm volatile( + "{\n\t" + ".reg .pred p, q;\n\t" + "setp.ne.b32 q, %0, 0;\n\t" + "setp.ne.b32 p, %5, 0;\n\t" + "@q tcgen05.mma.cta_group::1.kind::f16 [%1], %2, %3, %4, {%6, %7, %8, %9}, p;\n\t" + "}\n" + :: "r"(lead), "r"(tmem_c), "l"(desc_a), "l"(desc_b), "r"(idesc), + "r"(enable_input_d ? 1u : 0u), "r"(0u), "r"(0u), "r"(0u), "r"(0u)); +} + +__device__ __forceinline__ void tcgen05_mma_f16_ts_1sm_lead(uint32_t lead, + uint32_t tmem_c, uint32_t tmem_a, uint64_t desc_b, uint32_t idesc, + bool enable_input_d) { + asm volatile( + "{\n\t" + ".reg .pred p, q;\n\t" + "setp.ne.b32 q, %0, 0;\n\t" + "setp.ne.b32 p, %5, 0;\n\t" + "@q tcgen05.mma.cta_group::1.kind::f16 [%1], [%2], %3, %4, {%6, %7, %8, %9}, p;\n\t" + "}\n" + :: "r"(lead), "r"(tmem_c), "r"(tmem_a), "l"(desc_b), "r"(idesc), + "r"(enable_input_d ? 1u : 0u), "r"(0u), "r"(0u), "r"(0u), "r"(0u)); +} + +enum class SmemSwizzleBlackwell : uint32_t { + None = 0, + B128_32atom = 1, + B128 = 2, + B64 = 4, + B32 = 6, +}; + +__device__ __host__ __forceinline__ uint64_t build_smem_desc_blackwell( + uint32_t smem_addr, + uint32_t stride_byte_offset, + uint32_t leading_byte_offset, + SmemSwizzleBlackwell swizzle = SmemSwizzleBlackwell::B128, + uint32_t base_offset = 0) { + uint64_t d = 0; + d |= static_cast((smem_addr >> 4) & 0x3FFF); + d |= static_cast((leading_byte_offset >> 4) & 0x3FFF) << 16; + d |= static_cast((stride_byte_offset >> 4) & 0x3FFF) << 32; + d |= static_cast(1) << 46; + d |= (static_cast(base_offset) & 0x7) << 49; + d |= static_cast(static_cast(swizzle) & 0x7) << 61; + return d; +} + +__device__ __forceinline__ +uint32_t elect_one_sync() { + uint32_t elected; + asm volatile( + "{\n\t" + ".reg .pred p;\n\t" + "elect.sync %0|p, 0xffffffff;\n\t" + "selp.b32 %0, 1, 0, p;\n\t" + "}\n" + : "=r"(elected)); + return elected; +} + +__device__ __forceinline__ +uint32_t elect_one_sync(uint32_t membermask) { + uint32_t elected; + asm volatile( + "{\n\t" + ".reg .pred p;\n\t" + "elect.sync %0|p, %1;\n\t" + "selp.b32 %0, 1, 0, p;\n\t" + "}\n" + : "=r"(elected) : "r"(membermask)); + return elected; +} + +template +__device__ __forceinline__ +void setmaxnreg_dec() { + static_assert(N >= 24 && N <= 256 && N % 8 == 0, + "setmaxnreg_dec: N must be in [24, 256] and a multiple of 8"); + asm volatile("setmaxnreg.dec.sync.aligned.u32 %0;\n" + :: "n"(N) : "memory"); +} + +template +__device__ __forceinline__ +void setmaxnreg_inc() { + static_assert(N >= 24 && N <= 256 && N % 8 == 0, + "setmaxnreg_inc: N must be in [24, 256] and a multiple of 8"); + asm volatile("setmaxnreg.inc.sync.aligned.u32 %0;\n" + :: "n"(N) : "memory"); +} + +__device__ __forceinline__ +uint32_t cvt_f32x2_to_bf16x2(float a, float b) { + uint32_t r; + asm volatile("cvt.rn.bf16x2.f32 %0, %2, %1;\n" + : "=r"(r) : "f"(a), "f"(b)); + return r; +} + +namespace { +__device__ __forceinline__ uint64_t f32x2_bits(float2 v) { + uint64_t b; __builtin_memcpy(&b, &v, 8); return b; +} +__device__ __forceinline__ float2 f32x2_make(uint64_t b) { + float2 v; __builtin_memcpy(&v, &b, 8); return v; +} +} + +__device__ __forceinline__ float2 fmul2(float2 a, float2 b) { + uint64_t d; + asm volatile("mul.f32x2 %0, %1, %2;\n" : "=l"(d) : "l"(f32x2_bits(a)), "l"(f32x2_bits(b))); + return f32x2_make(d); +} + +__device__ __forceinline__ float2 fadd2(float2 a, float2 b) { + uint64_t d; + asm volatile("add.f32x2 %0, %1, %2;\n" : "=l"(d) : "l"(f32x2_bits(a)), "l"(f32x2_bits(b))); + return f32x2_make(d); +} + +__device__ __forceinline__ float2 ffma2(float2 a, float2 b, float2 c) { + uint64_t d; + asm volatile("fma.rn.f32x2 %0, %1, %2, %3;\n" + : "=l"(d) : "l"(f32x2_bits(a)), "l"(f32x2_bits(b)), "l"(f32x2_bits(c))); + return f32x2_make(d); +} + +__device__ __forceinline__ float2 f32x2_splat(float s) { return make_float2(s, s); } + +__device__ __forceinline__ float ex2_approx_f32(float z) { + float d; + asm volatile("ex2.approx.ftz.f32 %0, %1;\n" : "=f"(d) : "f"(z)); + return d; +} + +__device__ __forceinline__ float2 ex2_emu_f32x2(float x, float y) { + uint32_t ox, oy; + asm volatile( + "{\n\t" + ".reg .f32 f1,f2,f3,f4,f5,f6,f7;\n\t" + ".reg .b64 l1,l2,l3,l4,l5,l6,l7,l8,l9,l10;\n\t" + ".reg .s32 r1,r2,r3,r4,r5,r6,r7,r8;\n\t" + "max.f32 f1, %2, 0fC2FE0000;\n\t" + "max.f32 f2, %3, 0fC2FE0000;\n\t" + "mov.b64 l1, {f1, f2};\n\t" + "mov.f32 f3, 0f4B400000;\n\t" + "mov.b64 l2, {f3, f3};\n\t" + "add.rm.f32x2 l7, l1, l2;\n\t" + "sub.rn.f32x2 l8, l7, l2;\n\t" + "sub.rn.f32x2 l9, l1, l8;\n\t" + "mov.f32 f7, 0f3D9DF09D;\n\t" + "mov.b64 l6, {f7, f7};\n\t" + "mov.f32 f6, 0f3E6906A4;\n\t" + "mov.b64 l5, {f6, f6};\n\t" + "mov.f32 f5, 0f3F31F519;\n\t" + "mov.b64 l4, {f5, f5};\n\t" + "mov.f32 f4, 0f3F800000;\n\t" + "mov.b64 l3, {f4, f4};\n\t" + "fma.rn.f32x2 l10, l9, l6, l5;\n\t" + "fma.rn.f32x2 l10, l10, l9, l4;\n\t" + "fma.rn.f32x2 l10, l10, l9, l3;\n\t" + "mov.b64 {r1, r2}, l7;\n\t" + "mov.b64 {r3, r4}, l10;\n\t" + "shl.b32 r5, r1, 23;\n\t" + "add.s32 r7, r5, r3;\n\t" + "shl.b32 r6, r2, 23;\n\t" + "add.s32 r8, r6, r4;\n\t" + "mov.b32 %0, r7;\n\t" + "mov.b32 %1, r8;\n\t" + "}\n" + : "=r"(ox), "=r"(oy) : "f"(x), "f"(y)); + float2 r; __builtin_memcpy(&r.x, &ox, 4); __builtin_memcpy(&r.y, &oy, 4); + return r; +} + +__device__ __forceinline__ float rcp_approx_ftz_f32(float x) { + float d; + asm("rcp.approx.ftz.f32 %0, %1;\n" : "=f"(d) : "f"(x)); + return d; +} + +__device__ __forceinline__ uint32_t make_idesc_table44( + int M, int N, + uint32_t dtype, uint32_t atype, uint32_t btype, + bool transpose_a = false, bool transpose_b = false, + bool negate_a = false, bool negate_b = false) { + uint32_t idesc = 0; + idesc |= (dtype & 0x3) << 4; + idesc |= (atype & 0x7) << 7; + idesc |= (btype & 0x7) << 10; + idesc |= (negate_a ? 1u : 0u) << 13; + idesc |= (negate_b ? 1u : 0u) << 14; + idesc |= (transpose_a ? 1u : 0u) << 15; + idesc |= (transpose_b ? 1u : 0u) << 16; + idesc |= ((static_cast(N) >> 3) & 0x3F) << 17; + idesc |= ((static_cast(M) >> 4) & 0x1F) << 24; + return idesc; +} + +__device__ __forceinline__ uint32_t make_idesc_bf16_f32( + int M, int N, bool ta = false, bool tb = false) { + return make_idesc_table44(M, N, 1, + 1, 1, ta, tb); +} + +__device__ __forceinline__ void tcgen05_ld_32x32b_x16( + uint32_t tmem_addr, uint32_t (&r)[16]) { + asm volatile("tcgen05.ld.sync.aligned.32x32b.x16.b32 " + "{%0,%1,%2,%3,%4,%5,%6,%7,%8,%9," + "%10,%11,%12,%13,%14,%15}, [%16];\n" + : "=r"(r[0]),"=r"(r[1]),"=r"(r[2]),"=r"(r[3]), + "=r"(r[4]),"=r"(r[5]),"=r"(r[6]),"=r"(r[7]), + "=r"(r[8]),"=r"(r[9]),"=r"(r[10]),"=r"(r[11]), + "=r"(r[12]),"=r"(r[13]),"=r"(r[14]),"=r"(r[15]) + : "r"(tmem_addr)); +} + +__device__ __forceinline__ void tcgen05_ld_32x32b_x32( + uint32_t tmem_addr, uint32_t (&r)[32]) { + asm volatile("tcgen05.ld.sync.aligned.32x32b.x32.b32 " + "{%0,%1,%2,%3,%4,%5,%6,%7,%8,%9," + "%10,%11,%12,%13,%14,%15,%16,%17,%18,%19," + "%20,%21,%22,%23,%24,%25,%26,%27,%28,%29," + "%30,%31}, [%32];\n" + : "=r"(r[0]),"=r"(r[1]),"=r"(r[2]),"=r"(r[3]), + "=r"(r[4]),"=r"(r[5]),"=r"(r[6]),"=r"(r[7]), + "=r"(r[8]),"=r"(r[9]),"=r"(r[10]),"=r"(r[11]), + "=r"(r[12]),"=r"(r[13]),"=r"(r[14]),"=r"(r[15]), + "=r"(r[16]),"=r"(r[17]),"=r"(r[18]),"=r"(r[19]), + "=r"(r[20]),"=r"(r[21]),"=r"(r[22]),"=r"(r[23]), + "=r"(r[24]),"=r"(r[25]),"=r"(r[26]),"=r"(r[27]), + "=r"(r[28]),"=r"(r[29]),"=r"(r[30]),"=r"(r[31]) + : "r"(tmem_addr)); +} + +__device__ __forceinline__ void tcgen05_ld_32x32b_x64( + uint32_t tmem_addr, uint32_t (&r)[64]) { + asm volatile("tcgen05.ld.sync.aligned.32x32b.x64.b32 " + "{%0,%1,%2,%3,%4,%5,%6,%7,%8,%9," + "%10,%11,%12,%13,%14,%15,%16,%17,%18,%19," + "%20,%21,%22,%23,%24,%25,%26,%27,%28,%29," + "%30,%31,%32,%33,%34,%35,%36,%37,%38,%39," + "%40,%41,%42,%43,%44,%45,%46,%47,%48,%49," + "%50,%51,%52,%53,%54,%55,%56,%57,%58,%59," + "%60,%61,%62,%63}, [%64];\n" + : "=r"(r[0]),"=r"(r[1]),"=r"(r[2]),"=r"(r[3]), + "=r"(r[4]),"=r"(r[5]),"=r"(r[6]),"=r"(r[7]), + "=r"(r[8]),"=r"(r[9]),"=r"(r[10]),"=r"(r[11]), + "=r"(r[12]),"=r"(r[13]),"=r"(r[14]),"=r"(r[15]), + "=r"(r[16]),"=r"(r[17]),"=r"(r[18]),"=r"(r[19]), + "=r"(r[20]),"=r"(r[21]),"=r"(r[22]),"=r"(r[23]), + "=r"(r[24]),"=r"(r[25]),"=r"(r[26]),"=r"(r[27]), + "=r"(r[28]),"=r"(r[29]),"=r"(r[30]),"=r"(r[31]), + "=r"(r[32]),"=r"(r[33]),"=r"(r[34]),"=r"(r[35]), + "=r"(r[36]),"=r"(r[37]),"=r"(r[38]),"=r"(r[39]), + "=r"(r[40]),"=r"(r[41]),"=r"(r[42]),"=r"(r[43]), + "=r"(r[44]),"=r"(r[45]),"=r"(r[46]),"=r"(r[47]), + "=r"(r[48]),"=r"(r[49]),"=r"(r[50]),"=r"(r[51]), + "=r"(r[52]),"=r"(r[53]),"=r"(r[54]),"=r"(r[55]), + "=r"(r[56]),"=r"(r[57]),"=r"(r[58]),"=r"(r[59]), + "=r"(r[60]),"=r"(r[61]),"=r"(r[62]),"=r"(r[63]) + : "r"(tmem_addr)); +} + +__device__ __forceinline__ void tcgen05_ld_32x32b_x128( + uint32_t tmem_addr, uint32_t (&r)[128]) { + asm volatile("tcgen05.ld.sync.aligned.32x32b.x128.b32 " + "{%0,%1,%2,%3,%4,%5,%6,%7,%8,%9," + "%10,%11,%12,%13,%14,%15,%16,%17,%18,%19," + "%20,%21,%22,%23,%24,%25,%26,%27,%28,%29," + "%30,%31,%32,%33,%34,%35,%36,%37,%38,%39," + "%40,%41,%42,%43,%44,%45,%46,%47,%48,%49," + "%50,%51,%52,%53,%54,%55,%56,%57,%58,%59," + "%60,%61,%62,%63,%64,%65,%66,%67,%68,%69," + "%70,%71,%72,%73,%74,%75,%76,%77,%78,%79," + "%80,%81,%82,%83,%84,%85,%86,%87,%88,%89," + "%90,%91,%92,%93,%94,%95,%96,%97,%98,%99," + "%100,%101,%102,%103,%104,%105,%106,%107,%108,%109," + "%110,%111,%112,%113,%114,%115,%116,%117,%118,%119," + "%120,%121,%122,%123,%124,%125,%126,%127}, [%128];\n" + : "=r"(r[ 0]),"=r"(r[ 1]),"=r"(r[ 2]),"=r"(r[ 3]), + "=r"(r[ 4]),"=r"(r[ 5]),"=r"(r[ 6]),"=r"(r[ 7]), + "=r"(r[ 8]),"=r"(r[ 9]),"=r"(r[ 10]),"=r"(r[ 11]), + "=r"(r[ 12]),"=r"(r[ 13]),"=r"(r[ 14]),"=r"(r[ 15]), + "=r"(r[ 16]),"=r"(r[ 17]),"=r"(r[ 18]),"=r"(r[ 19]), + "=r"(r[ 20]),"=r"(r[ 21]),"=r"(r[ 22]),"=r"(r[ 23]), + "=r"(r[ 24]),"=r"(r[ 25]),"=r"(r[ 26]),"=r"(r[ 27]), + "=r"(r[ 28]),"=r"(r[ 29]),"=r"(r[ 30]),"=r"(r[ 31]), + "=r"(r[ 32]),"=r"(r[ 33]),"=r"(r[ 34]),"=r"(r[ 35]), + "=r"(r[ 36]),"=r"(r[ 37]),"=r"(r[ 38]),"=r"(r[ 39]), + "=r"(r[ 40]),"=r"(r[ 41]),"=r"(r[ 42]),"=r"(r[ 43]), + "=r"(r[ 44]),"=r"(r[ 45]),"=r"(r[ 46]),"=r"(r[ 47]), + "=r"(r[ 48]),"=r"(r[ 49]),"=r"(r[ 50]),"=r"(r[ 51]), + "=r"(r[ 52]),"=r"(r[ 53]),"=r"(r[ 54]),"=r"(r[ 55]), + "=r"(r[ 56]),"=r"(r[ 57]),"=r"(r[ 58]),"=r"(r[ 59]), + "=r"(r[ 60]),"=r"(r[ 61]),"=r"(r[ 62]),"=r"(r[ 63]), + "=r"(r[ 64]),"=r"(r[ 65]),"=r"(r[ 66]),"=r"(r[ 67]), + "=r"(r[ 68]),"=r"(r[ 69]),"=r"(r[ 70]),"=r"(r[ 71]), + "=r"(r[ 72]),"=r"(r[ 73]),"=r"(r[ 74]),"=r"(r[ 75]), + "=r"(r[ 76]),"=r"(r[ 77]),"=r"(r[ 78]),"=r"(r[ 79]), + "=r"(r[ 80]),"=r"(r[ 81]),"=r"(r[ 82]),"=r"(r[ 83]), + "=r"(r[ 84]),"=r"(r[ 85]),"=r"(r[ 86]),"=r"(r[ 87]), + "=r"(r[ 88]),"=r"(r[ 89]),"=r"(r[ 90]),"=r"(r[ 91]), + "=r"(r[ 92]),"=r"(r[ 93]),"=r"(r[ 94]),"=r"(r[ 95]), + "=r"(r[ 96]),"=r"(r[ 97]),"=r"(r[ 98]),"=r"(r[ 99]), + "=r"(r[100]),"=r"(r[101]),"=r"(r[102]),"=r"(r[103]), + "=r"(r[104]),"=r"(r[105]),"=r"(r[106]),"=r"(r[107]), + "=r"(r[108]),"=r"(r[109]),"=r"(r[110]),"=r"(r[111]), + "=r"(r[112]),"=r"(r[113]),"=r"(r[114]),"=r"(r[115]), + "=r"(r[116]),"=r"(r[117]),"=r"(r[118]),"=r"(r[119]), + "=r"(r[120]),"=r"(r[121]),"=r"(r[122]),"=r"(r[123]), + "=r"(r[124]),"=r"(r[125]),"=r"(r[126]),"=r"(r[127]) + : "r"(tmem_addr)); +} + +__device__ __forceinline__ +uint32_t smem_ptr_u32(const void* ptr) { + return static_cast(__cvta_generic_to_shared(ptr)); +} + +__device__ __forceinline__ +void sts_f32(uint32_t smem_addr, float val) { + asm volatile("st.shared.f32 [%0], %1;" :: "r"(smem_addr), "f"(val) : "memory"); +} diff --git a/python/sglang/kernels/ops/diffusion/__init__.py b/python/sglang/kernels/ops/diffusion/__init__.py index dd5569d5db..15d73e4b1f 100644 --- a/python/sglang/kernels/ops/diffusion/__init__.py +++ b/python/sglang/kernels/ops/diffusion/__init__.py @@ -252,6 +252,13 @@ _SPECS: tuple[tuple[str, KernelBackend, str, frozenset, str], ...] = ( _CUDA, "Fused in-place QK RMS-norm + RoPE.", ), + ( + "diffusion.vsa_block_sparse_sm100", + KernelBackend.JIT, + "attention.vsa_block_sparse_sm100_jit:vsa_block_sparse_sm100", + frozenset({CapabilityRequirement.cuda(min_sm=(10, 0), max_sm=(10, 3))}), + "FastVideo's warp-specialized tcgen05 block-sparse VSA forward (64-token tiles).", + ), ( "diffusion.h3_vae_rmsnorm", KernelBackend.TRITON, @@ -715,6 +722,8 @@ _EXPORTS: dict[str, str] = { "prepare_rope_tables": "attention.sana_wm_gdn_triton", "_attn_fwd": "attention.sparse_linear_attn_triton", "get_block_map": "attention.sparse_linear_attn_triton", + "can_use_vsa_block_sparse_sm100": "attention.vsa_block_sparse_sm100_jit", + "vsa_block_sparse_sm100": "attention.vsa_block_sparse_sm100_jit", # MoE routing "can_use_group_limited_topk": "routing.group_limited_topk_triton", "group_limited_topk": "routing.group_limited_topk_triton", diff --git a/python/sglang/kernels/ops/diffusion/attention/vsa_block_sparse_sm100_jit.py b/python/sglang/kernels/ops/diffusion/attention/vsa_block_sparse_sm100_jit.py new file mode 100644 index 0000000000..2c921eadad --- /dev/null +++ b/python/sglang/kernels/ops/diffusion/attention/vsa_block_sparse_sm100_jit.py @@ -0,0 +1,67 @@ +"""FastVideo's native Blackwell block-sparse VSA forward (64-token tiles). + +A CTA owns two adjacent query tiles, so the tile count must be even. Requires +sm_100a or sm_103a, bf16, head_dim 128 and contiguous [B, H, S, D]. +""" + +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING + +import torch + +from sglang.kernels.jit.utils import cache_once, load_jit +from sglang.srt.utils.custom_op import register_custom_op + +if TYPE_CHECKING: + from tvm_ffi.module import Module + +logger = logging.getLogger(__name__) + +_SUPPORTED_CAPABILITIES = ((10, 0), (10, 3)) +_HEAD_DIM = 128 + + +@cache_once +def _module() -> Module: + return load_jit( + "vsa_block_sparse_sm100", + cuda_files=["diffusion/vsa_block_sparse_sm100.cuh"], + cuda_wrappers=[("vsa_block_sparse_sm100", "VsaBlockSparseSm100Kernel::run")], + extra_ldflags=["-lcuda"], + ) + + +@cache_once +def can_use_vsa_block_sparse_sm100( + device_index: int, dtype: torch.dtype, head_dim: int +) -> bool: + if dtype != torch.bfloat16 or head_dim != _HEAD_DIM: + return False + if torch.cuda.get_device_capability(device_index) not in _SUPPORTED_CAPABILITIES: + return False + try: + _module() + except Exception as e: + logger.warning("Failed to load the JIT VSA block-sparse sm100 kernel: %s", e) + return False + return True + + +@register_custom_op(mutates_args=["out"]) +def vsa_block_sparse_sm100( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + q2k_idx: torch.Tensor, + q2k_num: torch.Tensor, + block_sizes: torch.Tensor, + out: torch.Tensor, + sm_scale: float, +) -> None: + """``out[b, h, i*64:(i+1)*64]`` = attention of query tile ``i`` over its + ``q2k_num`` listed key tiles, keys past each tile's ``block_sizes`` masked.""" + _module().vsa_block_sparse_sm100( + q, k, v, q2k_idx, q2k_num, block_sizes, out, sm_scale + ) diff --git a/python/sglang/multimodal_gen/.agents/skills/sglang-diffusion-benchmark-profile/benchmark-and-profile.md b/python/sglang/multimodal_gen/.agents/skills/sglang-diffusion-benchmark-profile/benchmark-and-profile.md index 7517c94f46..f672349cc8 100644 --- a/python/sglang/multimodal_gen/.agents/skills/sglang-diffusion-benchmark-profile/benchmark-and-profile.md +++ b/python/sglang/multimodal_gen/.agents/skills/sglang-diffusion-benchmark-profile/benchmark-and-profile.md @@ -314,7 +314,7 @@ Use the preset categories this way: | `cosmos3-super-t2v-cfg2tp2` | `nvidia/Cosmos3-Super` | No | Explicit four-GPU TP2 x CFG2 throughput comparator. On H200 it was 48.00% faster end to end than TP2, but the topology changed the deterministic output (SSIM 0.914244, PSNR 29.469771 dB), so do not treat it as lossless-equivalent or select it automatically. | | `wan-i2v` | `Wan-AI/Wan2.2-I2V-A14B-Diffusers` | Yes: `wan22_i2v_a14b_720p` | Nightly cat image and motion prompt, 1280x720, 81 frames, 4 GPUs, CFG parallel, Ulysses degree 2, text encoder CPU offload and pinned CPU memory | | `minimax-h3-t2va` | `MiniMaxAI/MiniMax-H3` | Yes: `minimax_h3_t2va_5s` | H3 FL2VA-partition T2VA baseline: 1344x768 resolved canvas, 5 seconds / 124 frames at 24 fps, 50 joint video-audio steps, 4 GPUs, TP2 + Ulysses2, eager BF16/FP32. The helper writes H3's request contract to a generated config. | -| `fasth3-t2va-vsa` | `FastVideo/FastVideo-FastH3-4-step-Preview-v1-VSA-DataFree` | No | FastH3 4-step distilled T2VA on the trained VSA-H3 backend: 1344x768, 10 seconds / 243 frames, five sigma points = four DiT forwards, 4 GPUs, Ulysses 4, eager, 2-step warmup request. Compare against `--attention-backend fa` on the same weights for the dense-fallback gap. | +| `fasth3-t2va-vsa` | `FastVideo/FastVideo-FastH3-8-Step-V2` | No | FastH3 8-step distilled T2VA on the trained VSA-H3 backend (sparsity 0.8): 1344x768, 10 seconds / 243 frames, nine sigma points = eight DiT forwards, 4 GPUs, Ulysses 4, eager, 2-step warmup request. Compare against `--attention-backend fa` on the same weights for the dense-fallback gap. | | `longcat-image` | `meituan-longcat/LongCat-Image` | No | Eager DiT baseline at 1024x1024, 50 steps, guidance 4.5; prompt rewrite is disabled so Qwen2.5-VL does not contaminate the DiT A/B. | | `longcat-image-edit` | `meituan-longcat/LongCat-Image-Edit` | No | Native edit baseline using the public SGLang edit fixture. Its 1536x1024 source resolves to 1264x848 under the checkpoint's roughly-one-megapixel aspect-ratio rule, and the BCG comparator captures that exact serving canvas; prompt rewrite is disabled to isolate the DiT. | | `longcat-image-edit-turbo` | `meituan-longcat/LongCat-Image-Edit-Turbo` | No | Matching distilled edit baseline using the same public fixture, prompt, and 1264x848 BCG canvas. Its registered sampling class owns the eight-step, guidance-1 schedule. | diff --git a/python/sglang/multimodal_gen/.agents/skills/sglang-diffusion-benchmark-profile/scripts/bench_diffusion_denoise.py b/python/sglang/multimodal_gen/.agents/skills/sglang-diffusion-benchmark-profile/scripts/bench_diffusion_denoise.py index d0c34cd88d..765eccafbe 100755 --- a/python/sglang/multimodal_gen/.agents/skills/sglang-diffusion-benchmark-profile/scripts/bench_diffusion_denoise.py +++ b/python/sglang/multimodal_gen/.agents/skills/sglang-diffusion-benchmark-profile/scripts/bench_diffusion_denoise.py @@ -427,7 +427,7 @@ MODELS = { }, # H3 rejects a 1-step warmup request, hence --warmup-steps=2. "fasth3-t2va-vsa": { - "path": "FastVideo/FastVideo-FastH3-4-step-Preview-v1-VSA-DataFree", + "path": "FastVideo/FastVideo-FastH3-8-Step-V2", "prompt": ( "A curious raccoon peers through a vibrant field of yellow " "sunflowers, its eyes wide with interest." @@ -441,12 +441,11 @@ MODELS = { "aspect_ratio": "16:9", "duration_seconds": 10.0, }, - "num_inference_steps": 5, + "num_inference_steps": 9, }, "extra_args": [ "--num-gpus=4", - "--attention-backend=video_sparse_attn_h3", - '--attention-backend-config={"VSA_sparsity": 0.9}', + "--component-attention-backends=transformer=video_sparse_attn_h3", "--enable-torch-compile=false", "--warmup-steps=2", ], diff --git a/python/sglang/multimodal_gen/README.md b/python/sglang/multimodal_gen/README.md index 24bbd84684..d4f8aac550 100644 --- a/python/sglang/multimodal_gen/README.md +++ b/python/sglang/multimodal_gen/README.md @@ -37,6 +37,9 @@ SGLang Diffusion supports Apple Silicon (M-series) via the MPS backend. Since Tr Docker is recommended for Linux GPU deployments. Follow the [installation guide](https://docs.sglang.io/docs/sglang-diffusion/installation) to start a container with diffusion dependencies included, or choose pip/uv, source installation, or another platform's setup. Run the examples below inside that environment. +Video output requires a system `ffmpeg` with the `libx264` encoder on `PATH`. For Ubuntu source or pip installations, install it with `sudo apt-get install ffmpeg`; on macOS, use `brew install ffmpeg`. + + ## Inference Here's a minimal example to generate a video using the default settings: diff --git a/python/sglang/multimodal_gen/configs/models/vaes/minimax_h3_video.py b/python/sglang/multimodal_gen/configs/models/vaes/minimax_h3_video.py index 0a790a7377..945381c895 100644 --- a/python/sglang/multimodal_gen/configs/models/vaes/minimax_h3_video.py +++ b/python/sglang/multimodal_gen/configs/models/vaes/minimax_h3_video.py @@ -33,6 +33,8 @@ class MiniMaxH3VideoVAEConfig(VAEConfig): load_decoder: bool = True use_tiling: bool = True use_parallel_tiling: bool = True + # Decode each rank's spatial tiles in one batched call instead of one by one. + stack_tiling: bool = False # The released checkpoint's quality contract uses overlapping latent # tiles. Parallel tiling distributes whole tiles without changing that # recipe. Spatial-shard decode is rejected because validation found output diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/minimax_h3.py b/python/sglang/multimodal_gen/configs/pipeline_configs/minimax_h3.py index 5d20b4730d..c99edc832c 100644 --- a/python/sglang/multimodal_gen/configs/pipeline_configs/minimax_h3.py +++ b/python/sglang/multimodal_gen/configs/pipeline_configs/minimax_h3.py @@ -57,6 +57,8 @@ class MiniMaxH3PipelineConfig(PipelineConfig): vae_precision: str = "fp32" vae_decode_precision: str = "fp16" audio_vae_precision: str = "fp32" + # Default VSA-H3 sparsity; --attention-backend-config VSA_sparsity overrides. + vsa_sparsity: float = 0.9 text_encoder_configs: tuple[MiniMaxH3Qwen3VLConfig, ...] = field( default_factory=lambda: (MiniMaxH3Qwen3VLConfig(),) ) @@ -333,16 +335,20 @@ class MiniMaxH3PipelineConfig(PipelineConfig): @dataclass class FastH3PipelineConfig(MiniMaxH3PipelineConfig): - """FastH3: 4-step VSA-distilled MiniMax-H3, t2va only.""" + """FastH3 8-Step V2: VSA-distilled MiniMax-H3, t2va only.""" + + # The checkpoint's trained sparsity (fastvideo_inference.json). + vsa_sparsity: float = 0.8 def __post_init__(self) -> None: self.dit_config.arch_config.has_gate_compress = True + self.vae_config.stack_tiling = True def validate_quality_deployment(self, server_args) -> None: raise ValueError( 'quality="high" is audited only for the base MiniMax-H3 50-step ' - "4xH200 deployment; the FastH3 4-step distilled checkpoint has no " - 'audited high-quality deployment. Use quality="exact".' + "4xH200 deployment; FastH3 has no audited high-quality deployment. " + 'Use quality="exact", or the default "lossless".' ) def validate_server_args(self, server_args) -> None: @@ -383,7 +389,7 @@ def register(): sampling_param_cls=FastH3SamplingParams, pipeline_config_cls=FastH3PipelineConfig, hf_model_paths=[ - "FastVideo/FastVideo-FastH3-4-step-Preview-v1-VSA-DataFree", + "FastVideo/FastVideo-FastH3-8-Step-V2", ], model_detectors=[ lambda model_id: ( diff --git a/python/sglang/multimodal_gen/configs/sample/minimax_h3.py b/python/sglang/multimodal_gen/configs/sample/minimax_h3.py index 42f90ac477..6b18ebfcf7 100644 --- a/python/sglang/multimodal_gen/configs/sample/minimax_h3.py +++ b/python/sglang/multimodal_gen/configs/sample/minimax_h3.py @@ -357,15 +357,16 @@ class MiniMaxH3SamplingParams(SamplingParams): @dataclass class FastH3SamplingParams(MiniMaxH3SamplingParams): - """FastH3: five sigma grid points, i.e. the four distilled DiT forwards.""" + """FastH3 8-Step V2: nine sigma points, i.e. eight DiT forwards on the trained rungs.""" - num_inference_steps: int = 5 + num_inference_steps: int = 9 + quality: str = "lossless" def _validate(self) -> None: super()._validate() - if self.num_inference_steps != 5: + if self.num_inference_steps != 9: raise ValueError( - "FastH3 is distilled for exactly five sigma grid points (four DiT " + "FastH3 is distilled for exactly nine sigma grid points (eight DiT " f"forwards); got num_inference_steps={self.num_inference_steps}. " "Use MiniMaxAI/MiniMax-H3 for other schedules." ) diff --git a/python/sglang/multimodal_gen/registry.py b/python/sglang/multimodal_gen/registry.py index d2b9b31a49..0f2a025a48 100644 --- a/python/sglang/multimodal_gen/registry.py +++ b/python/sglang/multimodal_gen/registry.py @@ -176,7 +176,7 @@ KNOWN_NON_DIFFUSERS_DIFFUSION_MODEL_PATTERNS: Dict[str, str] = { "ming-image-0.1-design": "MingImagePipeline", "minimaxai/minimax-h3": "MiniMaxH3Pipeline", "minimax/minimax-h3": "MiniMaxH3Pipeline", - "fastvideo/fastvideo-fasth3-4-step-preview-v1-vsa-datafree": "FastH3Pipeline", + "fastvideo/fastvideo-fasth3-8-step-v2": "FastH3Pipeline", "openvdn/vdn-minimax-h3": "VDNH3Pipeline", "lerobot/pi05": "Pi05Pipeline", "pi05": "Pi05Pipeline", diff --git a/python/sglang/multimodal_gen/runtime/entrypoints/utils.py b/python/sglang/multimodal_gen/runtime/entrypoints/utils.py index a64facc499..d520bc8624 100644 --- a/python/sglang/multimodal_gen/runtime/entrypoints/utils.py +++ b/python/sglang/multimodal_gen/runtime/entrypoints/utils.py @@ -13,7 +13,6 @@ import json import mmap import os import queue -import shutil import subprocess import tempfile import threading @@ -23,6 +22,7 @@ from dataclasses import dataclass, field from typing import Any, Callable, List, Optional, Sequence, Union import imageio +import imageio_ffmpeg import numpy as np import torch from PIL import Image @@ -32,11 +32,6 @@ try: except ImportError: # pragma: no cover scipy_wavfile = None -try: - import imageio_ffmpeg as _imageio_ffmpeg -except ImportError: # pragma: no cover - _imageio_ffmpeg = None - from sglang.multimodal_gen.configs.sample.sampling_params import ( DataType, SamplingParams, @@ -300,25 +295,8 @@ def _pick_audio_sample_rate( def _resolve_ffmpeg_exe() -> str: - ffmpeg_exe = "ffmpeg" - ffmpeg_on_path = shutil.which("ffmpeg") - if ffmpeg_on_path: - ffmpeg_exe = ffmpeg_on_path - try: - if _imageio_ffmpeg is not None: - ffmpeg_exe = _imageio_ffmpeg.get_ffmpeg_exe() - except Exception: - pass - - ffmpeg_ok = False - if ffmpeg_exe: - if os.path.isabs(ffmpeg_exe): - ffmpeg_ok = os.path.exists(ffmpeg_exe) - else: - ffmpeg_ok = shutil.which(ffmpeg_exe) is not None - if not ffmpeg_ok: - raise RuntimeError("ffmpeg not found") - return ffmpeg_exe + # keep the pinned encoder binary: x264 upgrades change lossy video pixels + return imageio_ffmpeg.get_ffmpeg_exe() # ffmpeg's implicit libx264 default is `medium`. On diffusion output `fast` is @@ -716,7 +694,7 @@ def _try_save_cuda_video_direct( if encoder is not None: encoder.abort() logger.warning_once( - "Direct CUDA video save failed; falling back to imageio. " + "Direct CUDA video save failed; falling back to CPU ffmpeg encoding. " "Enable debug logging for exception details." ) logger.debug("Direct CUDA video save failure", exc_info=True) @@ -894,6 +872,83 @@ def _maybe_mux_audio_into_mp4( ) +def _save_video_ffmpeg( + save_file_path: str, + frames: list, + *, + fps: int, + quality: float, + x264_preset: Optional[str] = None, + audio_path: Optional[str] = None, +) -> None: + if not frames: + raise ValueError("video output requires at least one frame") + if not 1 <= quality <= 10: + raise ValueError("video quality must be between 1 and 10") + first_frame = np.asarray(frames[0]) + height, width = first_frame.shape[:2] + channels = first_frame.shape[2] if first_frame.ndim == 3 else 1 + pixel_format = {1: "gray", 3: "rgb24", 4: "rgba"}[channels] + command = [ + _resolve_ffmpeg_exe(), + "-y", + "-f", + "rawvideo", + "-vcodec", + "rawvideo", + "-s", + f"{width}x{height}", + "-pix_fmt", + pixel_format, + "-r", + f"{fps:.02f}", + "-i", + "pipe:0", + ] + if audio_path is None: + command += ["-an"] + else: + command += ["-i", audio_path, "-acodec", "aac"] + command += [ + "-vcodec", + "libx264", + "-preset", + x264_preset or X264_PRESET, + "-pix_fmt", + "yuv420p", + "-crf", + str(int((1 - quality / 10.0) * 51)), + ] + if width % 16 or height % 16: + command += [ + "-vf", + f"scale={(width + 15) // 16 * 16}:{(height + 15) // 16 * 16}", + ] + command += [save_file_path] + with tempfile.TemporaryFile() as stderr: + with subprocess.Popen( + command, + stdin=subprocess.PIPE, + stdout=subprocess.DEVNULL, + stderr=stderr, + ) as process: + try: + for frame in frames: + frame = np.asarray(frame) + if frame.shape != first_frame.shape or frame.dtype != np.uint8: + raise ValueError( + "video frames must have matching shapes and uint8 dtype" + ) + process.stdin.write(memoryview(np.ascontiguousarray(frame))) + finally: + process.stdin.close() + if process.wait() != 0: + stderr.seek(0) + raise RuntimeError( + f"ffmpeg video encoding failed: {stderr.read().decode(errors='replace')}" + ) + + def _try_save_video_with_audio( *, save_file_path: str, @@ -901,7 +956,6 @@ def _try_save_video_with_audio( fps: int, audio: Any, audio_sample_rate: Optional[int], - output_format: str, quality: float, x264_preset: Optional[str] = None, ) -> bool: @@ -925,16 +979,13 @@ def _try_save_video_with_audio( with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as f: tmp_wav_path = f.name scipy_wavfile.write(tmp_wav_path, selected_sr, audio_np) - imageio.mimsave( + _save_video_ffmpeg( save_file_path, frames, fps=fps, - format=output_format, - codec="libx264", quality=quality, audio_path=tmp_wav_path, - audio_codec="aac", - output_params=["-preset", x264_preset or X264_PRESET], + x264_preset=x264_preset, ) return True except Exception as e: @@ -1151,26 +1202,22 @@ def save_materialized_output( os.makedirs(os.path.dirname(save_file_path), exist_ok=True) if data_type == DataType.VIDEO: quality = output_compression / 10 if output_compression is not None else 5 - output_format = data_type.get_default_extension() saved_with_audio = _try_save_video_with_audio( save_file_path=save_file_path, frames=materialized.frames, fps=materialized.fps, audio=materialized.audio, audio_sample_rate=audio_sample_rate, - output_format=output_format, quality=quality, x264_preset=x264_preset, ) if not saved_with_audio: - imageio.mimsave( + _save_video_ffmpeg( save_file_path, materialized.frames, fps=materialized.fps, - format=output_format, - codec="libx264", quality=quality, - output_params=["-preset", x264_preset or X264_PRESET], + x264_preset=x264_preset, ) _maybe_mux_audio_into_mp4( diff --git a/python/sglang/multimodal_gen/runtime/layers/attention/backends/video_sparse_attn_h3.py b/python/sglang/multimodal_gen/runtime/layers/attention/backends/video_sparse_attn_h3.py index 2a08122845..aefb70f046 100644 --- a/python/sglang/multimodal_gen/runtime/layers/attention/backends/video_sparse_attn_h3.py +++ b/python/sglang/multimodal_gen/runtime/layers/attention/backends/video_sparse_attn_h3.py @@ -29,6 +29,7 @@ from typing import Any import torch +from sglang.kernels.ops.diffusion import can_use_vsa_block_sparse_sm100 from sglang.multimodal_gen.runtime.layers.attention.backends.attention_backend import ( AttentionBackend, AttentionImpl, @@ -42,7 +43,9 @@ from sglang.multimodal_gen.runtime.layers.attention.backends.video_sparse_attn i ) from sglang.multimodal_gen.runtime.layers.attention.backends.vsa_h3_kernels import ( vsa_h3_block_sparse_attn_forward, + vsa_h3_gate_add, vsa_h3_pack_tiles, + vsa_h3_topk_lists, vsa_h3_untile, ) from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum @@ -238,6 +241,7 @@ def _topk_tile_lists( prefix = num_prefix_tiles keep = _compute_topk(sparsity, num_video_tiles) video_rows = scores[:, prefix:, :] + # serving takes vsa_h3_topk_lists for this branch; it stays as its reference if exempt or prefix == 0: picked = video_rows[:, :, prefix:].topk(keep, dim=-1).indices + prefix picked = picked.sort(dim=-1).values @@ -277,6 +281,8 @@ class _Workspace: """Per-geometry scratch: tiled q/k/v(/gate) [3|4, H, S_pad, D], pooled fp32 tile means [3, H, n_tiles, D], and the kernel index lists (prefix rows and prefix columns are static; only the top-k video columns change per layer). + The native kernel pairs query tiles, so the tile axis is padded to an even + count with one empty tile. """ def __init__( @@ -289,9 +295,14 @@ class _Workspace: device: torch.device, ) -> None: n_tiles = meta.num_tiles - seq_pad = n_tiles * VSA_H3_TILE_ELEMS + self.n_tiles = n_tiles + self.use_sm100_kernel = can_use_vsa_block_sparse_sm100( + device.index, dtype, head_dim + ) + n_tiles_padded = n_tiles + (n_tiles % 2 if self.use_sm100_kernel else 0) + seq_pad = n_tiles_padded * VSA_H3_TILE_ELEMS self.key = _workspace_key(meta, heads, head_dim, has_gate, dtype, device) - self.tiled = torch.empty( + self.tiled = torch.zeros( (3 + int(has_gate), heads, seq_pad, head_dim), dtype=dtype, device=device ) self.pooled = torch.empty( @@ -300,23 +311,67 @@ class _Workspace: self.out_tiled = torch.empty( (heads, seq_pad, head_dim), dtype=dtype, device=device ) + self.block_sizes = torch.zeros(n_tiles_padded, dtype=torch.int32, device=device) + self.block_sizes[:n_tiles] = meta.variable_block_sizes all_tiles = torch.arange(n_tiles, dtype=torch.int32, device=device) - self.dense_index = all_tiles.repeat(heads, n_tiles, 1) - self.dense_num = torch.full( - (heads, n_tiles), n_tiles, dtype=torch.int32, device=device + self.dense_index = torch.zeros( + (heads, n_tiles_padded, n_tiles), dtype=torch.int32, device=device ) + self.dense_index[:, :n_tiles] = all_tiles + self.dense_num = torch.zeros( + (heads, n_tiles_padded), dtype=torch.int32, device=device + ) + self.dense_num[:, :n_tiles] = n_tiles self.q2k_index = self.dense_index.clone() self.q2k_num = self.dense_num.clone() + num_video_tiles = meta.num_video_tiles + # Zeroed scratch for the exempt top-k compaction; kept zero between uses. + self.topk_mask = torch.zeros( + (heads, num_video_tiles, num_video_tiles), dtype=torch.int8, device=device + ) def sparse_lists( self, video_lists: torch.Tensor, num_prefix_tiles: int ) -> tuple[torch.Tensor, torch.Tensor]: width = video_lists.shape[-1] - self.q2k_index[:, num_prefix_tiles:, :width] = video_lists - self.q2k_num[:, num_prefix_tiles:] = width + self.q2k_index[:, num_prefix_tiles : self.n_tiles, :width] = video_lists + self.q2k_num[:, num_prefix_tiles : self.n_tiles] = width return self.q2k_index, self.q2k_num +def _gate_tile_index( + meta: VideoSparseAttentionH3Metadata, row_start: int, num_rows: int +) -> torch.Tensor: + """Tile of each packed row in the shard, -1 for pad rows.""" + key = ("gate_tile_index", row_start, num_rows) + tile_index = meta.workspace_cache.get(key) + if tile_index is None: + unpack_index = meta.unpack_index + rows = torch.arange(row_start, row_start + num_rows, device=unpack_index.device) + tile_index = torch.full_like(rows, -1, dtype=torch.int32) + in_used = rows < meta.total_seq_length + tile_index[in_used] = unpack_index[rows[in_used]] // VSA_H3_TILE_ELEMS + meta.workspace_cache[key] = tile_index + return tile_index + + +def vsa_h3_fold_gate( + out: torch.Tensor, + gate_compress: torch.Tensor, + out_compress: torch.Tensor, + meta: VideoSparseAttentionH3Metadata, + row_start: int, +) -> None: + """Fold the compression branch into ``out`` [rows, H, D], the row shard at + ``row_start``.""" + vsa_h3_gate_add( + out, + gate_compress, + out_compress, + _gate_tile_index(meta, row_start, out.shape[0]), + ) + + def _get_workspace( meta: VideoSparseAttentionH3Metadata, query: torch.Tensor, has_gate: bool ) -> _Workspace: @@ -341,6 +396,15 @@ def _select_kv_lists( keep = _compute_topk(sparsity, meta.num_video_tiles) if sparsity <= 0.0 or keep >= meta.num_video_tiles: return ws.dense_index, ws.dense_num + prefix = meta.num_prefix_tiles + if meta.exempt or prefix == 0: + # Same lists as _topk_tile_lists without sorting values or indices. + picked = scores[:, prefix:, prefix:].topk(keep, dim=-1, sorted=False).indices + vsa_h3_topk_lists( + picked, ws.topk_mask, ws.q2k_index[:, prefix : ws.n_tiles], prefix + ) + ws.q2k_num[:, prefix : ws.n_tiles] = prefix + keep + return ws.q2k_index, ws.q2k_num return ws.sparse_lists( _topk_tile_lists( scores, @@ -393,7 +457,9 @@ class VideoSparseAttentionH3Impl(AttentionImpl): attn_metadata: AttentionMetadata, ) -> torch.Tensor: raise NotImplementedError( - "VSA-H3 serves MiniMax-H3's packed varlen attention; use forward_varlen." + "VSA-H3 serves MiniMax-H3's packed varlen attention only; select it " + "for the transformer with --component-attention-backends " + "transformer=video_sparse_attn_h3 so dense layers keep their backends." ) def forward_varlen( @@ -407,8 +473,15 @@ class VideoSparseAttentionH3Impl(AttentionImpl): cu_seqlens_host: tuple[int, ...] | None = None, attn_metadata: VideoSparseAttentionH3Metadata | None = None, gate_compress: torch.Tensor | None = None, - ) -> torch.Tensor: - """query/key/value: [T, H, D] packed rows (post-norm, post-RoPE).""" + return_compress: bool = False, + ) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: + """query/key/value: [T, H, D] packed rows (post-norm, post-RoPE). + ``return_compress`` returns the compression branch's ``[H, n_tiles, D]`` + output instead of folding it, for ``vsa_h3_fold_gate`` after the + caller's collectives.""" + assert not return_compress or ( + gate_compress is None and attn_metadata is not None + ), "return_compress replaces gate_compress and needs VSA-H3 metadata" if self.layer_idx is None or attn_metadata is None: if attn_metadata is None and self.layer_idx is not None: raise RuntimeError( @@ -441,6 +514,7 @@ class VideoSparseAttentionH3Impl(AttentionImpl): sparsity = 0.0 if self.layer_idx in meta.dense_layers else meta.VSA_sparsity has_gate = gate_compress is not None + needs_compress = has_gate or return_compress ws = _get_workspace(meta, query, has_gate) vsa_h3_pack_tiles( @@ -456,7 +530,7 @@ class VideoSparseAttentionH3Impl(AttentionImpl): q_pooled, k_pooled, v_pooled = ws.pooled scores = None - if sparsity > 0.0 or has_gate: + if sparsity > 0.0 or needs_compress: scores = torch.matmul(q_pooled, k_pooled.transpose(-2, -1)) * ( self.head_size**-0.5 ) @@ -468,21 +542,24 @@ class VideoSparseAttentionH3Impl(AttentionImpl): ws.tiled[2:3], q2k_index[None], q2k_num[None], - meta.variable_block_sizes, + ws.block_sizes, out=ws.out_tiled[None], + use_sm100_kernel=ws.use_sm100_kernel, ) out_compress = None - if has_gate: + if needs_compress: out_compress = torch.matmul(torch.softmax(scores, dim=-1), v_pooled) result = torch.empty(query.shape, dtype=query.dtype, device=query.device) vsa_h3_untile( ws.out_tiled, ws.tiled[3] if has_gate else None, - out_compress, + out_compress if has_gate else None, meta.unpack_index, used, result, ) + if return_compress: + return result, out_compress return result diff --git a/python/sglang/multimodal_gen/runtime/layers/attention/backends/vsa_h3_kernels.py b/python/sglang/multimodal_gen/runtime/layers/attention/backends/vsa_h3_kernels.py index e809cc8f1f..ab7c44dff4 100644 --- a/python/sglang/multimodal_gen/runtime/layers/attention/backends/vsa_h3_kernels.py +++ b/python/sglang/multimodal_gen/runtime/layers/attention/backends/vsa_h3_kernels.py @@ -17,6 +17,7 @@ padded tile layout the attention kernel reads and pools each tile in the same pass; ``vsa_h3_untile`` scatters the attention output back to packed rows and folds in the gated compression branch. Both replace chains of index copies and transposes that otherwise cost more than the attention kernel itself. +``vsa_h3_gate_add`` folds the same branch after the caller's collectives. """ import math @@ -26,6 +27,8 @@ import triton import triton.language as tl from triton.tools.tensor_descriptor import TensorDescriptor +from sglang.kernels.ops.diffusion import vsa_block_sparse_sm100 + # BLOCK_M / BLOCK_N are structural, not tunable: the kernel indexes the top-k # list per BLOCK_M q-tile and addresses keys as kv_idx * BLOCK_N, so both must # match the granularity q2k_index and variable_block_sizes were built at. @@ -106,10 +109,12 @@ def vsa_h3_block_sparse_attn_forward( q2k_num: torch.Tensor, variable_block_sizes: torch.Tensor, out: torch.Tensor | None = None, + use_sm100_kernel: bool = False, ) -> torch.Tensor: """q/k/v: contiguous [B, H, S_pad, D] bf16 with S_pad = n_tiles * 64; pad rows zero. q2k_index/q2k_num: contiguous [B, H, n_tiles, max_kv] / - [B, H, n_tiles] int32.""" + [B, H, n_tiles] int32. ``use_sm100_kernel`` runs FastVideo's tcgen05 kernel (sm_100a / + sm_103a, even n_tiles) instead of the Triton kernel.""" batch, heads, seq_q, head_dim = q.shape seq_kv = k.shape[2] if seq_q % VSA_H3_KERNEL_BLOCK or seq_kv % VSA_H3_KERNEL_BLOCK: @@ -124,6 +129,18 @@ def vsa_h3_block_sparse_attn_forward( ) if out is None: out = torch.empty_like(q) + if use_sm100_kernel: + vsa_block_sparse_sm100( + q, + k, + v, + q2k_index.view(-1, q2k_index.shape[-1]), + q2k_num.view(-1), + variable_block_sizes, + out, + 1.0 / math.sqrt(head_dim), + ) + return out block = [1, 1, VSA_H3_KERNEL_BLOCK, head_dim] desc_q, desc_k, desc_v, desc_o = ( TensorDescriptor.from_tensor(t, block_shape=block) for t in (q, k, v, out) @@ -243,9 +260,10 @@ def vsa_h3_pack_tiles( """Gather packed [T, H, D] rows into ``tiled`` [3|4, H, S_pad, D] and write fp32 per-tile means of q/k/v into ``pooled`` [3, H, n_tiles, D]. ``src_index`` maps each padded position to its packed row, or -1 (pad -> 0). + S_pad may hold one extra empty tile past ``n_tiles``; it is left untouched. """ _, heads, seq_pad, head_dim = tiled.shape - n_tiles = seq_pad // VSA_H3_KERNEL_BLOCK + n_tiles = pooled.shape[2] has_gate = gate is not None g = gate if has_gate else q assert all(t.stride(-1) == 1 for t in (q, k, v, g)) @@ -327,6 +345,8 @@ def vsa_h3_untile( heads, seq_pad, head_dim = out_tiled.shape total = result.shape[0] has_gate = gate_tiled is not None + # S_pad may carry one padded tile; the compression output does not + n_tiles = out_compress.shape[1] if has_gate else seq_pad // VSA_H3_KERNEL_BLOCK _untile_kernel[(triton.cdiv(total, VSA_H3_KERNEL_BLOCK), heads)]( out_tiled, gate_tiled if has_gate else out_tiled, @@ -336,8 +356,126 @@ def vsa_h3_untile( used, total, seq_pad, - seq_pad // VSA_H3_KERNEL_BLOCK, + n_tiles, HAS_GATE=has_gate, HEAD_DIM=head_dim, BLOCK=VSA_H3_KERNEL_BLOCK, ) + + +@triton.jit +def _gate_add_kernel( + Out, + Gate, + Compress, + tile_index, + num_rows, + stride_o_row, + stride_o_head, + stride_g_row, + stride_g_head, + N_TILES, + HEAD_DIM: tl.constexpr, + BLOCK: tl.constexpr, +): + row_block = tl.program_id(0) + h = tl.program_id(1) + rows = row_block * BLOCK + tl.arange(0, BLOCK) + cols = tl.arange(0, HEAD_DIM) + in_rows = rows < num_rows + tile = tl.load(tile_index + rows, mask=in_rows, other=-1) + valid = (tile >= 0)[:, None] + o_off = ( + rows.to(tl.int64)[:, None] * stride_o_row + h * stride_o_head + cols[None, :] + ) + g_off = ( + rows.to(tl.int64)[:, None] * stride_g_row + h * stride_g_head + cols[None, :] + ) + o = tl.load(Out + o_off, mask=valid, other=0.0).to(tl.float32) + g = tl.load(Gate + g_off, mask=valid, other=0.0).to(tl.float32) + c = tl.load( + Compress + (h * N_TILES + tile)[:, None] * HEAD_DIM + cols[None, :], + mask=valid, + other=0.0, + ) + o = o + c * g + tl.store(Out + o_off, o.to(Out.type.element_ty), mask=valid) + + +def vsa_h3_gate_add( + out: torch.Tensor, + gate: torch.Tensor, + out_compress: torch.Tensor, + tile_index: torch.Tensor, +) -> None: + """In place: ``out[t, h] += out_compress[h, tile_index[t]] * gate[t, h]`` + where ``tile_index[t] >= 0``; same fp32 arithmetic as ``vsa_h3_untile``.""" + num_rows, heads, head_dim = out.shape + assert out.stride(-1) == 1 and gate.stride(-1) == 1 + assert out_compress.is_contiguous() and out_compress.shape[0] == heads + _gate_add_kernel[(triton.cdiv(num_rows, VSA_H3_KERNEL_BLOCK), heads)]( + out, + gate, + out_compress, + tile_index, + num_rows, + out.stride(0), + out.stride(1), + gate.stride(0), + gate.stride(1), + out_compress.shape[1], + HEAD_DIM=head_dim, + BLOCK=VSA_H3_KERNEL_BLOCK, + ) + + +@triton.jit +def _topk_lists_kernel( + mask_ptr, + out_ptr, + num_video_tiles, + prefix, + stride_mh, + stride_mr, + stride_oh, + stride_or, + VIDEO_POW2: tl.constexpr, +): + row = tl.program_id(0) + head = tl.program_id(1) + mask_row = mask_ptr + head * stride_mh + row * stride_mr + out_row = out_ptr + head * stride_oh + row * stride_or + cols = tl.arange(0, VIDEO_POW2) + in_range = cols < num_video_tiles + picked = tl.load(mask_row + cols, mask=in_range, other=0).to(tl.int32) + slot = tl.cumsum(picked, axis=0) - 1 + tl.store(out_row + prefix + slot, cols + prefix, mask=in_range & (picked != 0)) + # Leave the mask zeroed for the next layer's scatter. + tl.store(mask_row + cols, tl.zeros_like(picked).to(tl.int8), mask=in_range) + + +def vsa_h3_topk_lists( + picked: torch.Tensor, mask: torch.Tensor, out: torch.Tensor, num_prefix_tiles: int +) -> None: + """Write ``sort(picked + prefix)`` into ``out[h, r, prefix:]``. + + ``picked`` [H, rows, keep] holds distinct video-tile indices in any order; + ``mask`` [H, rows, num_video_tiles] is an all-zero int8 scratch, left + zeroed; ``out`` [H, rows, >= prefix + keep] is int32 whose first ``prefix`` + columns already hold ``[0, prefix)``.""" + heads, rows, _ = picked.shape + num_video_tiles = mask.shape[-1] + assert mask.dtype == torch.int8 and mask.is_contiguous() + assert out.dtype == torch.int32 and out.stride(-1) == 1 + mask.scatter_(-1, picked, 1) + _topk_lists_kernel[(rows, heads)]( + mask, + out, + num_video_tiles, + num_prefix_tiles, + mask.stride(0), + mask.stride(1), + out.stride(0), + out.stride(1), + VIDEO_POW2=triton.next_power_of_2(num_video_tiles), + ) diff --git a/python/sglang/multimodal_gen/runtime/layers/usp.py b/python/sglang/multimodal_gen/runtime/layers/usp.py index 794bec9c37..35dc951245 100644 --- a/python/sglang/multimodal_gen/runtime/layers/usp.py +++ b/python/sglang/multimodal_gen/runtime/layers/usp.py @@ -109,6 +109,20 @@ def _usp_all_to_all_single(x: torch.Tensor, role: str | None = None) -> torch.Te return output.reshape(x_shape) +def _usp_all_gather(x: torch.Tensor) -> torch.Tensor: + """Concatenate ``x`` from every Ulysses rank along dim 0, rank order.""" + ulysses_pg = get_sp_group().ulysses_group + assert ulysses_pg is not None, "Ulysses process group is not initialized." + x = x.contiguous() + output = torch.empty( + (get_ulysses_parallel_world_size() * x.shape[0], *x.shape[1:]), + dtype=x.dtype, + device=x.device, + ) + dist.all_gather_into_tensor(output, x, group=ulysses_pg) + return output + + def _usp_all_to_all_single_varlen( x: torch.Tensor, output_split_sizes: list[int], diff --git a/python/sglang/multimodal_gen/runtime/models/dits/minimax_h3.py b/python/sglang/multimodal_gen/runtime/models/dits/minimax_h3.py index 2443c16e91..782b79a223 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/minimax_h3.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/minimax_h3.py @@ -59,6 +59,9 @@ from sglang.multimodal_gen.runtime.distributed.parallel_state import ( from sglang.multimodal_gen.runtime.layers.attention.backends.attention_backend import ( AttentionRequirements, ) +from sglang.multimodal_gen.runtime.layers.attention.backends.video_sparse_attn_h3 import ( + vsa_h3_fold_gate, +) from sglang.multimodal_gen.runtime.layers.attention.selector import ( claim_deferred_component_attn_backend, get_attn_backend, @@ -673,14 +676,12 @@ def _minimax_h3_attention_core_impl( if ulysses_active: from sglang.multimodal_gen.runtime.layers.usp import ( - _usp_input_all_to_all, + _usp_all_gather, _usp_input_all_to_all_packed_qkv, _usp_output_all_to_all, ) q, k, v = _usp_input_all_to_all_packed_qkv(q, k, v) - if gate_compress is not None: - gate_compress = _usp_input_all_to_all(gate_compress[None], head_dim=2)[0] if attention._attention_impl is None: attention._set_attention_backend( @@ -698,7 +699,9 @@ def _minimax_h3_attention_core_impl( if attention.prefix.startswith("blocks.") else None ) - out = attention._attention_impl.forward_varlen( + # the gate stays on this rank's row shard; fold it after the output all-to-all + fold_after_exchange = ulysses_active and gate_compress is not None + attn_result = attention._attention_impl.forward_varlen( q, k, v, @@ -706,10 +709,24 @@ def _minimax_h3_attention_core_impl( max_seqlen=max_seqlen, cu_seqlens_host=cu_seqlens_host, attn_metadata=attn_metadata, - gate_compress=gate_compress, + gate_compress=None if fold_after_exchange else gate_compress, + return_compress=fold_after_exchange, ) + if fold_after_exchange: + out, out_compress = attn_result + else: + out = attn_result if ulysses_active: out = _usp_output_all_to_all(out[None], head_dim=2)[0] + if fold_after_exchange: + _, ulysses_rank = get_ulysses_ctx() + vsa_h3_fold_gate( + out, + gate_compress, + _usp_all_gather(out_compress), + attn_metadata, + row_start=ulysses_rank * out.shape[0], + ) return out if ring_active: diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3.py b/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3.py index 26f256b47b..ed6bfab525 100644 --- a/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3.py +++ b/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3.py @@ -73,6 +73,7 @@ class MiniMaxH3VideoVAE(AutoencoderKLLegacy, LayerwiseOffloadableModuleMixin): and config.use_parallel_decode and config.use_parallel_tiling and bool(arch.vae_parallel_tiling), + stack_tiling=config.stack_tiling, tile_size=int(arch.vae_tile_size), tile_overlap_min=int(arch.vae_tile_overlap_min), encoder_parallel=False, diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_vae_cuda_opt.py b/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_vae_cuda_opt.py new file mode 100644 index 0000000000..105cfba393 --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_vae_cuda_opt.py @@ -0,0 +1,181 @@ +# SPDX-License-Identifier: Apache-2.0 +"""CUDA fast path for the MiniMax-H3 video VAE decoder (ViT3DDecoder). + +Fuses each decoder attention block's per-head QK RMSNorm + NeoX RoPE into one +in-place launch over the strided Q/K views of the QKV buffer and runs the +block's attention on cuDNN SDPA. Forwards are bound once at VAE load and +dispatch on a decode-scoped :class:`VaeFastPathGate`: ``quality="lossless"`` +and ``"high"`` take the fast path (rounding-level differences), the +``"exact"`` runs the original module path bit-for-bit. Install is +all-or-nothing and fail-closed. +""" + +from types import MethodType + +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.nn.attention import SDPBackend, sdpa_kernel + +from sglang.kernels.ops.diffusion import ( + can_use_fused_inplace_qknorm_rope, + fused_inplace_qknorm_rope, +) +from sglang.multimodal_gen.runtime.models.vaes.fast_path_gate import ( + VaeFastPathGate, + register_vae_fast_path_gate, +) +from sglang.multimodal_gen.runtime.models.vaes.minimax_h3_video_vae.attention import ( + Attention, + _apply_qk_norm, + _sdpa_attention, +) +from sglang.multimodal_gen.runtime.models.vaes.minimax_h3_video_vae.vit_utils import ( + _env_flag, + apply_rotary_pos_emb_qk, + native_rope_cache, +) +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +logger = init_logger(__name__) + + +def _fused_qknorm_rope(self, query, key, rotary_pos_emb) -> bool: + native = native_rope_cache(query, key, rotary_pos_emb) + if native is None: + return False + cache, positions = native + if not can_use_fused_inplace_qknorm_rope( + head_dim=self.dim_head, + rope_dim=cache.shape[1], + is_neox=True, + dtype=query.dtype, + cache_dtype=cache.dtype, + round_norm_before_rope=True, + ): + return False + weight = self._sgl_unit_weight + if weight is None or weight.dtype != query.dtype or weight.device != query.device: + weight = torch.ones(self.dim_head, dtype=query.dtype, device=query.device) + self._sgl_unit_weight = weight + # Batched decoder tiles flatten into rows; a view, so the update is in place. + fused_inplace_qknorm_rope( + query.flatten(0, 1), + key.flatten(0, 1), + weight, + weight, + cache, + positions, + is_neox=True, + eps=self.norm_q.eps, + head_dim=self.dim_head, + rope_dim=cache.shape[1], + round_norm_before_rope=True, + ) + return True + + +def _cudnn_attention(self, query, key, value): + if not self._sgl_cudnn_failed: + try: + with sdpa_kernel(SDPBackend.CUDNN_ATTENTION): + return F.scaled_dot_product_attention( + query.transpose(1, 2), key.transpose(1, 2), value.transpose(1, 2) + ).transpose(1, 2) + except RuntimeError as e: + logger.warning( + "MiniMax-H3 VAE: cuDNN SDPA failed (%s); using the layer's " + "attention backend.", + e, + ) + self._sgl_cudnn_failed = True + return self.attn(query, key, value) + + +def _attn_fast_forward(self, hidden_states, rotary_pos_emb=None): + if ( + not self._sgl_gate.enabled + or rotary_pos_emb is None + or torch.is_grad_enabled() + or torch.compiler.is_compiling() + ): + return type(self).forward(self, hidden_states, rotary_pos_emb) + + batch_size, seq_len, _ = hidden_states.shape + qkv = self.to_qkv(hidden_states) + qkv = qkv.view(batch_size, seq_len, -1, 3 * self.dim_head) + query, key, value = torch.chunk(qkv, 3, dim=-1) + + if not _fused_qknorm_rope(self, query, key, rotary_pos_emb): + query = _apply_qk_norm(self.norm_q, query) + key = _apply_qk_norm(self.norm_k, key) + query, key = apply_rotary_pos_emb_qk(query, key, rotary_pos_emb) + + if self.attn is not None and query.dtype in (torch.float16, torch.bfloat16): + hidden_states = _cudnn_attention(self, query, key, value) + else: + hidden_states = _sdpa_attention(query, key, value) + + hidden_states = hidden_states.reshape(batch_size, seq_len, -1) + return self.to_out(hidden_states) + + +def _plain_rms_norm(module) -> bool: + return isinstance(module, nn.RMSNorm) and module.weight is None + + +def _attn_fast_compatible(attn: nn.Module) -> bool: + return ( + type(attn) is Attention + and _plain_rms_norm(attn.norm_q) + and _plain_rms_norm(attn.norm_k) + and attn.norm_q.eps == attn.norm_k.eps + ) + + +def _install_fast_attention( + attn_modules: list[nn.Module], gate: VaeFastPathGate +) -> None: + for attn in attn_modules: + attn._sgl_gate = gate + attn._sgl_unit_weight = None + attn._sgl_cudnn_failed = False + attn.forward = MethodType(_attn_fast_forward, attn) + + +def maybe_optimize_minimax_h3_vae(vae: nn.Module) -> nn.Module: + """Install the quality-gated CUDA MiniMax-H3 VAE decoder fast path.""" + from sglang.multimodal_gen.runtime.models.vaes.minimax_h3 import MiniMaxH3VideoVAE + from sglang.multimodal_gen.runtime.models.vaes.minimax_h3_video_vae.vae_vit import ( + ViT3DDecoder, + ) + + if not isinstance(vae, MiniMaxH3VideoVAE): + return vae + decoder = vae.decoder + if type(decoder) is not ViT3DDecoder: + return vae + if not _env_flag("MINIMAX_H3_VAE_DECODER_VIT_FP32_NORM", "1"): + logger.info("MiniMax-H3 VAE: fp32 QK norm disabled; skipping fast path.") + return vae + + attn_modules = [block.attn for block in decoder.transformer_blocks] + eligible = [attn for attn in attn_modules if _attn_fast_compatible(attn)] + if len(eligible) != len(attn_modules): + logger.warning( + "MiniMax-H3 VAE: %d/%d decoder attention blocks non-standard; " + "skipping fast path.", + len(attn_modules) - len(eligible), + len(attn_modules), + ) + return vae + + gate = VaeFastPathGate() + _install_fast_attention(eligible, gate) + register_vae_fast_path_gate(vae, gate) + logger.info( + "MiniMax-H3 VAE: installed quality-gated fast path (%d QK RMSNorm+RoPE " + "fusions, cuDNN SDPA).", + len(eligible), + ) + return vae diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/vae_vit.py b/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/vae_vit.py index fafb9433c6..6e1272c538 100644 --- a/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/vae_vit.py +++ b/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/vae_vit.py @@ -426,6 +426,7 @@ class ViT3DDecoder(ViTBase): rotary_pos_emb = prepare_rotary_pos_emb( self.pos_embed(img_ids[:1]), dtype=rotary_dtype, + batch=B, ) if B > 1: rotary_pos_emb = _expand_rotary_batch(rotary_pos_emb, B) diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/vit_utils.py b/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/vit_utils.py index c40e1baaa0..e777fa8d27 100644 --- a/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/vit_utils.py +++ b/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/vit_utils.py @@ -115,8 +115,10 @@ def prepare_rotary_pos_emb( rotary_pos_emb: Tuple[torch.Tensor, torch.Tensor], *, dtype: torch.dtype, + batch: int = 1, ) -> tuple[torch.Tensor, ...]: - """Prebuild the native Q/K rotary cache once per ViT decoder forward.""" + """Prebuild the native Q/K rotary cache once per ViT decoder forward; its + positions cover ``batch`` samples sharing one token grid.""" cos, sin = rotary_pos_emb if ( not cos.is_cuda @@ -143,7 +145,7 @@ def prepare_rotary_pos_emb( cos.shape[1], dtype=torch.long, device=cos.device, - ) + ).repeat(batch) return cos, sin, cache, positions @@ -209,13 +211,15 @@ def apply_rotary_pos_emb( raise -def apply_rotary_pos_emb_qk( +def native_rope_cache( query: torch.Tensor, key: torch.Tensor, rotary_pos_emb: Sequence[torch.Tensor], -) -> tuple[torch.Tensor, torch.Tensor]: - """Apply the exact native NeoX rotary kernel to Q/K together when possible.""" - if ( +) -> tuple[torch.Tensor, torch.Tensor] | None: + """``(cache, positions)`` from ``prepare_rotary_pos_emb`` when the native + kernels can consume it for this Q/K, else None. ``positions`` covers the + batch-major rows of ``query.flatten(0, 1)``.""" + if not ( len(rotary_pos_emb) == 4 and query.is_cuda and query.shape == key.shape @@ -224,33 +228,47 @@ def apply_rotary_pos_emb_qk( and query.dim() == 4 and not torch.compiler.is_compiling() ): - _, _, cache, positions = rotary_pos_emb - batch, seq_len = query.shape[:2] - if ( - cache.is_cuda - and cache.dtype == query.dtype - and cache.dim() == 2 - and cache.shape[0] == seq_len - and cache.shape[1] <= query.shape[-1] - and positions.is_cuda - and positions.shape == (seq_len,) - ): - from sgl_kernel import rotary_embedding + return None + _, _, cache, positions = rotary_pos_emb + if not ( + cache.is_cuda + and cache.dtype == query.dtype + and cache.dim() == 2 + and cache.shape[0] == query.shape[1] + and cache.shape[1] <= query.shape[-1] + and positions.is_cuda + and positions.shape in ((query.shape[1],), (query.shape[0] * query.shape[1],)) + ): + return None + # Accept both main's shared sequence positions and the pre-expanded + # batch-major positions used by the decoder cache and fused fast path. + if positions.shape == (query.shape[1],) and query.shape[0] > 1: + positions = positions.repeat(query.shape[0]) + return cache, positions - # Stacked spatial tiles share one rotary table. Flatten the - # batch and replay that table once per tile. - query = query.contiguous() - key = key.contiguous() - token_positions = positions if batch == 1 else positions.repeat(batch) - rotary_embedding( - token_positions, - query.view(batch * seq_len, -1), - key.view(batch * seq_len, -1), - query.shape[-1], - cache, - True, - ) - return query, key + +def apply_rotary_pos_emb_qk( + query: torch.Tensor, + key: torch.Tensor, + rotary_pos_emb: Sequence[torch.Tensor], +) -> tuple[torch.Tensor, torch.Tensor]: + """Apply the exact native NeoX rotary kernel to Q/K together when possible.""" + native = native_rope_cache(query, key, rotary_pos_emb) + if native is not None: + from sgl_kernel import rotary_embedding + + cache, positions = native + query = query.contiguous() + key = key.contiguous() + rotary_embedding( + positions, + query.view(positions.shape[0], -1), + key.view(positions.shape[0], -1), + query.shape[-1], + cache, + True, + ) + return query, key return ( apply_rotary_pos_emb(query, rotary_pos_emb), diff --git a/python/sglang/multimodal_gen/runtime/pipelines/minimax_h3_pipeline.py b/python/sglang/multimodal_gen/runtime/pipelines/minimax_h3_pipeline.py index 7f5783b67f..96862da90a 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines/minimax_h3_pipeline.py +++ b/python/sglang/multimodal_gen/runtime/pipelines/minimax_h3_pipeline.py @@ -175,6 +175,11 @@ class MiniMaxH3Pipeline(LoRAPipeline, ComposedPipelineBase): self.add_stage( MiniMaxH3TimestepPreparationStage( sigma_shift_scales=sigma_shift_scales, + dmd_denoising_steps=( + release_metadata.dmd_denoising_steps + if release_metadata is not None + else None + ), ) ) self.add_stage( @@ -201,7 +206,7 @@ class MiniMaxH3Pipeline(LoRAPipeline, ComposedPipelineBase): class FastH3Pipeline(MiniMaxH3Pipeline): - """FastH3: 4-step DMD2-distilled MiniMax-H3 (t2va only). + """FastH3: 8-step DMD2-distilled MiniMax-H3 (V2, t2va only). The flat single-partition repo is materialized into the base-H3 layout by the bundled model overlay (see model_overlays/), so every stage, loader, diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/release_metadata.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/release_metadata.py index 030764effd..e33c19a8a5 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/release_metadata.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/release_metadata.py @@ -52,6 +52,8 @@ class MiniMaxH3ReleaseMetadata: task_aliases: Mapping[str, str] video_sigma_shift: float audio_sigma_shift: float + # trained DMD rungs of a distilled release, unshifted; None keeps the uniform grid + dmd_denoising_steps: tuple[int, ...] | None = None @classmethod def from_model_index( @@ -92,6 +94,21 @@ class MiniMaxH3ReleaseMetadata: "model_index.json._minimax_h3.sigma_shift_scales requires numeric " "video and audio values" ) from exc + dmd_steps = raw.get("dmd_denoising_steps") + if dmd_steps is not None: + if ( + not isinstance(dmd_steps, list) + or not dmd_steps + or any( + type(step) is not int or not 0 < step <= 1000 for step in dmd_steps + ) + or any(left <= right for left, right in zip(dmd_steps, dmd_steps[1:])) + ): + raise ValueError( + "model_index.json._minimax_h3.dmd_denoising_steps must be " + "strictly decreasing integers in (0, 1000]" + ) + dmd_steps = tuple(dmd_steps) metadata = cls( schema_version=1, partition=partition, @@ -99,6 +116,7 @@ class MiniMaxH3ReleaseMetadata: task_aliases=dict(aliases), video_sigma_shift=video_sigma, audio_sigma_shift=audio_sigma, + dmd_denoising_steps=dmd_steps, ) for task in metadata.tasks: if canonical_minimax_h3_task(task) != task: diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/decoding.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/decoding.py index 7a5c0052c3..95335800c3 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/decoding.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/decoding.py @@ -7,6 +7,9 @@ from contextlib import contextmanager, nullcontext import torch +from sglang.multimodal_gen.configs.sample.sampling_params import ( + quality_allows, +) from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType from sglang.multimodal_gen.runtime.distributed import ( get_replica_group, @@ -16,6 +19,7 @@ from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_c from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import ( ComponentUse, ) +from sglang.multimodal_gen.runtime.models.vaes.fast_path_gate import use_vae_fast_path from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import OutputBatch, Req from sglang.multimodal_gen.runtime.pipelines_core.stages.base import ( StageParallelismType, @@ -494,7 +498,13 @@ class MiniMaxH3DecodingStage(DecodingStage): ) decode_kwargs["on_frames"] = stream try: - with set_forward_context(current_timestep=0, attn_metadata=None): + with ( + use_vae_fast_path( + selected_video_vae, + quality_allows(batch.sampling_params.quality, "lossless"), + ), + set_forward_context(current_timestep=0, attn_metadata=None), + ): decoded = video_decode(visual_decode_latent, **decode_kwargs) if stream is not None: output_file_paths = stream.finish() diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/denoising.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/denoising.py index 544e8e8c90..f09ee81187 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/denoising.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/denoising.py @@ -1077,7 +1077,10 @@ def _maybe_prepare_vsa_h3_step_metadata( "VSA-H3 in SGLang serves the trained 64-token (4, 4, 4) tile " f"geometry; got vsa_tile_size={tile_size}." ) - sparsity = float(config.get("VSA_sparsity", config.get("sparsity", 0.9))) + default_sparsity = server_args.pipeline_config.vsa_sparsity + sparsity = float( + config.get("VSA_sparsity", config.get("sparsity", default_sparsity)) + ) if not 0.0 <= sparsity < 1.0: raise ValueError(f"VSA sparsity must be in [0, 1), got {sparsity}") mode = str(config.get("vsa_mode", "exempt")) diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/timestep_preparation.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/timestep_preparation.py index e9da0a80d4..fc5f6fa3ec 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/timestep_preparation.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/timestep_preparation.py @@ -26,12 +26,17 @@ class MiniMaxH3TimestepPreparationStage(PipelineStage): deduplicated_tensor_tree_output_fields = ("timesteps", "sigmas") deduplicated_extra_tensor_tree_output_keys = (MINIMAX_H3_SIGMAS_EXTRA_KEY,) - def __init__(self, sigma_shift_scales=None) -> None: + def __init__( + self, + sigma_shift_scales=None, + dmd_denoising_steps: tuple[int, ...] | None = None, + ) -> None: super().__init__() # Per-model sigma shift override (model_index.json "_minimax_h3" release # block, sigma_shift_scales): the schedule constants are a MODEL # serving contract — fl2va and ref2va use video 12 / audio 3 by default. self.sigma_shift_scales = sigma_shift_scales + self.dmd_denoising_steps = dmd_denoising_steps self._pdd_config = None pdd_heads = envs.SGLANG_DIFFUSION_MINIMAX_H3_PDD_HEADS if pdd_heads: @@ -197,12 +202,20 @@ class MiniMaxH3TimestepPreparationStage(PipelineStage): task_default=plan.default_audio_flow_shift, ), } + dmd_steps = self.dmd_denoising_steps + num_steps = requested_num_steps + if dmd_steps is not None and batch.is_warmup: + # warmup truncates the trained rungs instead of rescaling a shorter grid + num_steps = len(dmd_steps) + 1 sigmas: dict[str, list[float]] = {} for modality in ("video", "audio"): sigmas[modality] = minimax_h3_time_shift_sigmas( - num_steps=requested_num_steps, + num_steps=num_steps, shift_scale=scales[modality], + dmd_steps=dmd_steps, ) + if num_steps != requested_num_steps: + sigmas[modality] = sigmas[modality][: max(2, requested_num_steps)] batch.extra[MINIMAX_H3_SIGMAS_EXTRA_KEY] = sigmas def verify_input(self, batch: Req, server_args: ServerArgs) -> VerificationResult: diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/time_request.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/time_request.py index e6fb5f3921..3ba9f92014 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/time_request.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/time_request.py @@ -33,6 +33,7 @@ def minimax_h3_time_shift_sigmas( *, num_steps: int = 50, shift_scale: float = 6.0, + dmd_steps: tuple[int, ...] | None = None, ) -> list[float]: if shift_scale <= 0: raise ValueError("MiniMax H3 shift_scale must be > 0") @@ -41,14 +42,26 @@ def minimax_h3_time_shift_sigmas( import torch - # The rectified-flow sigma range is fixed at [1.0, 0.0]. - base = torch.linspace( - 1.0, - 0.0, - int(num_steps), - device="cpu", - dtype=torch.float32, - ) + if dmd_steps is not None: + if int(num_steps) != len(dmd_steps) + 1: + raise ValueError( + f"this checkpoint is distilled for {len(dmd_steps)} DiT forwards " + f"({len(dmd_steps) + 1} sigma grid points); got " + f"num_inference_steps={num_steps}" + ) + # trained rungs are unshifted timesteps over 1000; append the final step to sigma 0 + base = torch.tensor( + [step / 1000.0 for step in dmd_steps] + [0.0], dtype=torch.float32 + ) + else: + # The rectified-flow sigma range is fixed at [1.0, 0.0]. + base = torch.linspace( + 1.0, + 0.0, + int(num_steps), + device="cpu", + dtype=torch.float32, + ) shifted = float(shift_scale) * base / (1 + (float(shift_scale) - 1) * base) shifted, _ = torch.unique_consecutive(shifted, return_counts=True) # A one-point request is still exactly one point. Normal serving uses diff --git a/python/sglang/multimodal_gen/runtime/platforms/cuda.py b/python/sglang/multimodal_gen/runtime/platforms/cuda.py index d6227398f7..f713786715 100644 --- a/python/sglang/multimodal_gen/runtime/platforms/cuda.py +++ b/python/sglang/multimodal_gen/runtime/platforms/cuda.py @@ -847,8 +847,8 @@ class CudaPlatformBase(Platform): @classmethod def optimize_vae(cls, vae: torch.nn.Module) -> torch.nn.Module: - """Install the quality-gated FLUX.2 / AutoencoderKL / Wan / Qwen-Image - VAE decoder fast paths. + """Install the quality-gated FLUX.2 / AutoencoderKL / Wan / Qwen-Image / + MiniMax-H3 VAE decoder fast paths. Requests with quality="lossless" or "high" run the fast paths; the "exact" default runs the original module path bit-for-bit. See @@ -859,6 +859,9 @@ class CudaPlatformBase(Platform): maybe_optimize_autoencoder_kl, maybe_optimize_flux2_vae, ) + from sglang.multimodal_gen.runtime.models.vaes.minimax_h3_vae_cuda_opt import ( + maybe_optimize_minimax_h3_vae, + ) from sglang.multimodal_gen.runtime.models.vaes.wan_vae_cuda_opt import ( maybe_optimize_qwen_image_vae, maybe_optimize_wan_vae, @@ -868,6 +871,7 @@ class CudaPlatformBase(Platform): vae = maybe_optimize_autoencoder_kl(vae) vae = maybe_optimize_wan_vae(vae) vae = maybe_optimize_qwen_image_vae(vae) + vae = maybe_optimize_minimax_h3_vae(vae) except Exception: logger.warning( "Failed to apply CUDA VAE optimizations; using the unmodified VAE.", diff --git a/python/sglang/multimodal_gen/runtime/utils/model_overlay.py b/python/sglang/multimodal_gen/runtime/utils/model_overlay.py index 18515523c9..8c049c3e07 100644 --- a/python/sglang/multimodal_gen/runtime/utils/model_overlay.py +++ b/python/sglang/multimodal_gen/runtime/utils/model_overlay.py @@ -45,9 +45,9 @@ BUILTIN_MODEL_OVERLAY_REGISTRY: dict[str, dict[str, Any]] = { "overlay_repo_id": "AgainstEntropy/SANA-WM_streaming-overlay", "overlay_revision": "62c6840871ecc3559189047513ba0670e1bf62e7", }, - "FastVideo/FastVideo-FastH3-4-step-Preview-v1-VSA-DataFree": { - "overlay_repo_id": "kevin-mi/FastH3-4step-Preview-overlay", - "overlay_revision": "f769cb8001dae335089de7b250364335bc7cb183", + "FastVideo/FastVideo-FastH3-8-Step-V2": { + "overlay_repo_id": "kevin-mi/FastH3-8step-V2-overlay", + "overlay_revision": "308e2dd627218c40ae759a93745429800771a625", }, "OpenVDN/vdn-minimax-h3": { "overlay_repo_id": "kevin-mi/VDN-H3-overlay", diff --git a/python/sglang/multimodal_gen/test/server/gpu_cases.py b/python/sglang/multimodal_gen/test/server/gpu_cases.py index 8746bdfcbe..b0b82a5534 100644 --- a/python/sglang/multimodal_gen/test/server/gpu_cases.py +++ b/python/sglang/multimodal_gen/test/server/gpu_cases.py @@ -775,14 +775,12 @@ MINIMAX_H3_FOUR_GPU_H100_CASES = [ DiffusionTestCase( "fasth3_t2va_vsa_4gpu_h100", DiffusionServerArgs( - model_path="FastVideo/FastVideo-FastH3-4-step-Preview-v1-VSA-DataFree", + model_path="FastVideo/FastVideo-FastH3-8-Step-V2", modality="video", num_gpus=4, extras=[ - "--attention-backend", - "video_sparse_attn_h3", - "--attention-backend-config", - '{"VSA_sparsity": 0.9}', + "--component-attention-backends", + "transformer=video_sparse_attn_h3", "--enable-torch-compile", "false", ], @@ -805,7 +803,7 @@ MINIMAX_H3_FOUR_GPU_H100_CASES = [ "aspect_ratio": "16:9", "duration_seconds": 5.0, }, - "num_inference_steps": 5, + "num_inference_steps": 9, "seed": 42, }, ), diff --git a/python/sglang/multimodal_gen/test/unit/test_minimax_h3_fasth3.py b/python/sglang/multimodal_gen/test/unit/test_minimax_h3_fasth3.py index cf79081ccc..89ca14c2b0 100644 --- a/python/sglang/multimodal_gen/test/unit/test_minimax_h3_fasth3.py +++ b/python/sglang/multimodal_gen/test/unit/test_minimax_h3_fasth3.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -"""FastH3 (4-step VSA-distilled MiniMax-H3) registration and admission contracts.""" +"""FastH3 8-Step V2 (VSA-distilled MiniMax-H3) registration and admission contracts.""" from __future__ import annotations @@ -25,11 +25,18 @@ from sglang.multimodal_gen.runtime.distributed.parallel_state import ( from sglang.multimodal_gen.runtime.layers.linear import UnquantizedLinearMethod from sglang.multimodal_gen.runtime.layers.quantization.fp8 import Fp8Config from sglang.multimodal_gen.runtime.models.dits.minimax_h3 import MiniMaxH3DiTModel +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.release_metadata import ( + MiniMaxH3ReleaseMetadata, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.time_request import ( + minimax_h3_time_shift_sigmas, +) from sglang.multimodal_gen.test.single_test_file.component_accuracy.utils import ( ensure_distributed_env_defaults, ) -FASTH3_MODEL_ID = "FastVideo/FastVideo-FastH3-4-step-Preview-v1-VSA-DataFree" +FASTH3_MODEL_ID = "FastVideo/FastVideo-FastH3-8-Step-V2" +DMD_STEPS = (999, 874, 749, 624, 500, 375, 250, 125) def _ensure_single_process_parallel_runtime() -> None: @@ -45,18 +52,20 @@ def test_registry_resolves_fasth3_configs() -> None: assert info.pipeline_config_cls is FastH3PipelineConfig assert get_non_diffusers_pipeline_name(FASTH3_MODEL_ID) == "FastH3Pipeline" materialized = ( - "/cache/materialized_models/" - "FastVideo__FastVideo-FastH3-4-step-Preview-v1-VSA-DataFree-0123abcd" + "/cache/materialized_models/FastVideo__FastVideo-FastH3-8-Step-V2-0123abcd" ) assert get_model_info(materialized).sampling_param_cls is FastH3SamplingParams def test_fasth3_sampling_defaults_and_task_rejection() -> None: params = FastH3SamplingParams(prompt="p") - assert params.num_inference_steps == 5 + assert params.num_inference_steps == 9 assert params.guidance_scale == 1.0 + assert params.quality == "lossless" + assert FastH3SamplingParams(prompt="p", quality="extra-high").quality == "lossless" + assert FastH3SamplingParams(prompt="p", quality="exact").quality == "exact" - with pytest.raises(ValueError, match="exactly five sigma grid points"): + with pytest.raises(ValueError, match="exactly nine sigma grid points"): FastH3SamplingParams(prompt="p", num_inference_steps=50) with pytest.raises(ValueError, match="distilled for t2va only"): @@ -72,8 +81,56 @@ def test_fasth3_sampling_defaults_and_task_rejection() -> None: ) +def test_fasth3_dmd_schedule_matches_trained_rungs() -> None: + """The distilled rungs are shifted exactly once, as FastVideo serves them, and + a wrong step count or non-decreasing rungs are rejected.""" + metadata = MiniMaxH3ReleaseMetadata.from_model_index( + { + "_minimax_h3": { + "schema_version": 1, + "partition": "fl2va", + "tasks": ["t2va"], + "sigma_shift_scales": {"video": 10.0, "audio": 3.0}, + "dmd_denoising_steps": list(DMD_STEPS), + } + } + ) + assert metadata.dmd_denoising_steps == DMD_STEPS + # FastVideo's _set_dmd_schedule output for the video / audio shifts 10 / 3 + served = { + 10.0: [ + *(0.9999, 0.985788, 0.967575, 0.943168, 0.909091), + *(0.857143, 0.769231, 0.588235, 0.0), + ], + 3.0: [ + *(0.999666, 0.954148, 0.89952, 0.83274, 0.75), + *(0.642857, 0.5, 0.3, 0.0), + ], + } + for shift, expected in served.items(): + sigmas = minimax_h3_time_shift_sigmas( + num_steps=9, shift_scale=shift, dmd_steps=DMD_STEPS + ) + assert sigmas == pytest.approx(expected, abs=1e-6) + with pytest.raises(ValueError, match="8 DiT forwards"): + minimax_h3_time_shift_sigmas(num_steps=5, shift_scale=10.0, dmd_steps=DMD_STEPS) + with pytest.raises(ValueError, match="strictly decreasing"): + MiniMaxH3ReleaseMetadata.from_model_index( + { + "_minimax_h3": { + "schema_version": 1, + "partition": "fl2va", + "tasks": ["t2va"], + "sigma_shift_scales": {"video": 10.0, "audio": 3.0}, + "dmd_denoising_steps": [999, 999], + } + } + ) + + def test_fasth3_pipeline_config_gates_and_rejections() -> None: config = FastH3PipelineConfig() + assert config.vsa_sparsity == 0.8 assert config.dit_config.arch_config.has_gate_compress assert not MiniMaxH3PipelineConfig().dit_config.arch_config.has_gate_compress mapping = config.dit_config.arch_config.param_names_mapping diff --git a/python/sglang/multimodal_gen/test/unit/test_minimax_h3_vae_parallel_modes.py b/python/sglang/multimodal_gen/test/unit/test_minimax_h3_vae_parallel_modes.py index 3647ff63a4..787c9fc6a8 100644 --- a/python/sglang/multimodal_gen/test/unit/test_minimax_h3_vae_parallel_modes.py +++ b/python/sglang/multimodal_gen/test/unit/test_minimax_h3_vae_parallel_modes.py @@ -5,6 +5,7 @@ import os import subprocess import sys import textwrap +from types import SimpleNamespace from unittest import mock import pytest @@ -14,10 +15,17 @@ import torch.nn as nn from sglang.multimodal_gen.configs.models.vaes.minimax_h3_video import ( MiniMaxH3VideoVAEConfig, ) +from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context +from sglang.multimodal_gen.runtime.models.vaes.fast_path_gate import VaeFastPathGate from sglang.multimodal_gen.runtime.models.vaes.minimax_h3 import MiniMaxH3VideoVAE from sglang.multimodal_gen.runtime.models.vaes.minimax_h3_audio_vae.audio_vae import ( CausalAttention, ) +from sglang.multimodal_gen.runtime.models.vaes.minimax_h3_vae_cuda_opt import ( + _attn_fast_compatible, + _fused_qknorm_rope, + _install_fast_attention, +) from sglang.multimodal_gen.runtime.models.vaes.minimax_h3_video_vae import ( AutoencoderKLLegacy, ) @@ -25,14 +33,23 @@ from sglang.multimodal_gen.runtime.models.vaes.minimax_h3_video_vae.attention im Attention, _apply_qk_norm, ) +from sglang.multimodal_gen.runtime.models.vaes.minimax_h3_video_vae.base_module import ( + RotaryEmbeddingND, +) from sglang.multimodal_gen.runtime.models.vaes.minimax_h3_video_vae.vae_vit import ( ViT3DDecoder, ) from sglang.multimodal_gen.runtime.models.vaes.minimax_h3_video_vae.vit_utils import ( apply_rotary_pos_emb_qk, + create_token_ids, + prepare_rotary_pos_emb, ) from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum +requires_cuda = pytest.mark.skipif( + not torch.cuda.is_available(), reason="fused qk-norm+RoPE kernel needs CUDA" +) + def _init_kwargs(config: MiniMaxH3VideoVAEConfig): with mock.patch.object( @@ -97,6 +114,77 @@ def test_vit_qk_norm_supports_affine_free_rmsnorm(): assert output.shape == hidden_states.shape +@requires_cuda +def test_vit_fast_path_is_gated_and_matches_reference(): + """Gate closed: bit-identical to the original forward. Gate open: the fused + qk-norm+RoPE kernel and cuDNN SDPA run and match to rounding level.""" + device = torch.device("cuda") + dtype = torch.float16 + heads, dim_head, rope_dim = 4, 64, 48 + attention = Attention( + heads=heads, dim_head=dim_head, qk_norm_type="rms_norm", eps=1e-5 + ).to(device=device, dtype=dtype) + assert _attn_fast_compatible(attention) + + pos_embed = RotaryEmbeddingND(rope_dim, 100.0, n_dim=3, use_angle=True).to(device) + ids = create_token_ids((2, 4, 4), device, dtype) + ids = torch.cat([ids, torch.zeros((1, 5, 3), device=device, dtype=dtype)], 1) + rotary = prepare_rotary_pos_emb(pos_embed(ids), dtype=dtype) + generator = torch.Generator(device="cpu").manual_seed(0) + hidden = torch.randn((1, ids.shape[1], heads * dim_head), generator=generator) + hidden = hidden.to(device=device, dtype=dtype) + + with ( + torch.no_grad(), + torch.autocast("cuda", dtype=dtype), + set_forward_context(current_timestep=0, attn_metadata=None), + ): + reference = attention(hidden, rotary) + gate = VaeFastPathGate() + _install_fast_attention([attention], gate) + assert torch.equal(attention(hidden, rotary), reference) + gate.enabled = True + fused = attention(hidden, rotary) + assert attention._sgl_unit_weight is not None, "fused qk-norm+RoPE did not run" + assert attention._sgl_cudnn_failed is False, "cuDNN SDPA fell back" + torch.testing.assert_close(fused, reference, atol=2e-2, rtol=1e-2) + + +@requires_cuda +def test_vit_rope_batched_tiles_match_per_tile(): + """Stacked decoder tiles share one RoPE row; the fused (quality) and native + (exact) RoPE kernels over the flattened batch are bit-identical to one + tile at a time.""" + device, dtype = torch.device("cuda"), torch.float16 + tiles, heads, dim_head, rope_dim = 3, 4, 64, 48 + pos_embed = RotaryEmbeddingND(rope_dim, 100.0, n_dim=3, use_angle=True).to(device) + ids = create_token_ids((2, 4, 4), device, dtype) + rotary = prepare_rotary_pos_emb(pos_embed(ids), dtype=dtype) + batched_rotary = prepare_rotary_pos_emb(pos_embed(ids), dtype=dtype, batch=tiles) + generator = torch.Generator(device="cpu").manual_seed(0) + qkv = torch.randn( + (tiles, ids.shape[1], heads, 3 * dim_head), generator=generator + ).to(device=device, dtype=dtype) + attn = SimpleNamespace( + dim_head=dim_head, _sgl_unit_weight=None, norm_q=SimpleNamespace(eps=1e-5) + ) + + per_tile = qkv.clone() + for tile in range(tiles): + query, key, _ = per_tile[tile : tile + 1].chunk(3, dim=-1) + assert _fused_qknorm_rope(attn, query, key, rotary) + query, key, _ = qkv.chunk(3, dim=-1) + assert _fused_qknorm_rope(attn, query, key, batched_rotary) + assert torch.equal(qkv, per_tile) + + query, key, _ = qkv.chunk(3, dim=-1) + batched = apply_rotary_pos_emb_qk(query, key, batched_rotary) + for tile in range(tiles): + query, key, _ = qkv[tile : tile + 1].chunk(3, dim=-1) + for got, want in zip(batched, apply_rotary_pos_emb_qk(query, key, rotary)): + assert torch.equal(got[tile : tile + 1], want) + + def test_audio_vae_attention_defaults_to_local_sdpa_and_allows_fa(): class RecordingFA(nn.Module): backend = AttentionBackendEnum.FA diff --git a/python/sglang/multimodal_gen/test/unit/test_output_saving.py b/python/sglang/multimodal_gen/test/unit/test_output_saving.py index ee5bc9178e..500c01c49a 100644 --- a/python/sglang/multimodal_gen/test/unit/test_output_saving.py +++ b/python/sglang/multimodal_gen/test/unit/test_output_saving.py @@ -1,3 +1,5 @@ +import shutil + import numpy as np import pytest import torch @@ -100,15 +102,15 @@ def test_video_with_audio_uses_single_pass_encoder(tmp_path, monkeypatch): def write(*_args, **_kwargs): pass - def mimsave_spy(path, frames, **kwargs): + def encode_spy(path, frames, **kwargs): calls.append((path, frames, kwargs)) assert kwargs["audio_path"].endswith(".wav") - assert kwargs["audio_codec"] == "aac" + assert kwargs["quality"] == 5 def fail_legacy_mux(**_kwargs): raise AssertionError("the two-pass mux path should not run") - monkeypatch.setattr(output_utils.imageio, "mimsave", mimsave_spy) + monkeypatch.setattr(output_utils, "_save_video_ffmpeg", encode_spy) monkeypatch.setattr(output_utils, "scipy_wavfile", FakeWavFile) monkeypatch.setattr(output_utils, "_maybe_mux_audio_into_mp4", fail_legacy_mux) @@ -144,13 +146,13 @@ def test_x264_preset_reaches_every_video_save_path(tmp_path, monkeypatch): presets.append(("direct", kwargs["x264_preset"])) return False - def mimsave_spy(_path, _frames, **kwargs): - encoder = "single-pass" if "audio_path" in kwargs else "imageio" - presets.append((encoder, kwargs["output_params"][-1])) + def encode_spy(_path, _frames, **kwargs): + encoder = "single-pass" if "audio_path" in kwargs else "ffmpeg" + presets.append((encoder, kwargs["x264_preset"])) monkeypatch.setattr(output_utils, "_try_save_cuda_videos_direct", parallel_save) monkeypatch.setattr(output_utils, "_try_save_cuda_video_direct", direct_save) - monkeypatch.setattr(output_utils.imageio, "mimsave", mimsave_spy) + monkeypatch.setattr(output_utils, "_save_video_ffmpeg", encode_spy) monkeypatch.setattr(output_utils, "scipy_wavfile", FakeWavFile) output_utils.save_outputs( @@ -177,7 +179,7 @@ def test_x264_preset_reaches_every_video_save_path(tmp_path, monkeypatch): assert {encoder for encoder, _ in presets} == { "parallel", "direct", - "imageio", + "ffmpeg", "single-pass", } assert {preset for _, preset in presets} == {"ultrafast"} @@ -193,12 +195,12 @@ def test_video_audio_single_pass_failure_falls_back(tmp_path, monkeypatch): def write(*_args, **_kwargs): pass - def mimsave_spy(path, frames, **kwargs): + def encode_spy(path, frames, **kwargs): calls.append((path, frames, kwargs)) if "audio_path" in kwargs: raise RuntimeError("unsupported audio input") - monkeypatch.setattr(output_utils.imageio, "mimsave", mimsave_spy) + monkeypatch.setattr(output_utils, "_save_video_ffmpeg", encode_spy) monkeypatch.setattr(output_utils, "scipy_wavfile", FakeWavFile) monkeypatch.setattr( output_utils, @@ -316,3 +318,11 @@ def test_multiple_videos_use_parallel_direct_save_with_serial_fallback( assert kwargs["fps"] == 24 assert len(serial_calls) == 1 assert serial_calls[0]["save_file_path"] == paths[1] + + +def test_video_output_preserves_pinned_encoder(monkeypatch): + monkeypatch.setattr(shutil, "which", lambda _: "/system/ffmpeg") + monkeypatch.setattr( + output_utils.imageio_ffmpeg, "get_ffmpeg_exe", lambda: "/pinned/ffmpeg" + ) + assert output_utils._resolve_ffmpeg_exe() == "/pinned/ffmpeg" diff --git a/python/sglang/multimodal_gen/test/unit/test_video_encode_preset.py b/python/sglang/multimodal_gen/test/unit/test_video_encode_preset.py index 39d132c751..f1632b4da9 100644 --- a/python/sglang/multimodal_gen/test/unit/test_video_encode_preset.py +++ b/python/sglang/multimodal_gen/test/unit/test_video_encode_preset.py @@ -7,6 +7,7 @@ catches the failure this guards against -- the preset never being passed, and ffmpeg silently applying its own default. """ +import json import shutil import subprocess @@ -14,11 +15,17 @@ import numpy as np import pytest import torch +import sglang.multimodal_gen.runtime.entrypoints.utils as output_utils from sglang.multimodal_gen.configs.sample.sampling_params import ( DataType, SamplingParams, ) -from sglang.multimodal_gen.runtime.entrypoints.utils import X264_PRESET, save_outputs +from sglang.multimodal_gen.runtime.entrypoints.utils import ( + X264_PRESET, + MaterializedOutput, + save_materialized_output, + save_outputs, +) FPS = 8 FRAMES = 8 @@ -97,7 +104,7 @@ def test_saved_video_carries_the_configured_preset(tmp_path): @pytest.mark.parametrize( "device", [ - # a host tensor saves through imageio, a CUDA one through the ffmpeg pipe + # host and CUDA tensors use the same system ffmpeg encoder "cpu", pytest.param( "cuda", @@ -166,3 +173,75 @@ def test_saved_video_decodes_to_every_frame(tmp_path): codec, count = probe.split(",") assert codec == "h264" assert int(count) == FRAMES + + +@pytest.mark.skipif(shutil.which("ffmpeg") is None, reason="needs ffmpeg") +@pytest.mark.parametrize("with_audio", [False, True]) +@pytest.mark.parametrize("compression", [10, 100]) +def test_materialized_video_uses_ffmpeg(tmp_path, monkeypatch, with_audio, compression): + def reject_imageio(*args, **kwargs): + raise AssertionError("video output must use ffmpeg directly") + + monkeypatch.setattr(output_utils.imageio, "mimsave", reject_imageio) + rng = np.random.default_rng(3) + # non-contiguous RGB and non-macroblock dimensions exercise the raw pipe and scaling + frames = rng.integers(0, 256, (FRAMES, 30, 34, 3), dtype=np.uint8)[:, :, ::-1] + saved = tmp_path / "materialized.mp4" + save_materialized_output( + MaterializedOutput( + sample=None, + frames=list(frames), + audio=np.zeros((32000, 2), dtype=np.float32) if with_audio else None, + fps=FPS, + ), + DataType.VIDEO, + str(saved), + audio_sample_rate=32000, + output_compression=compression, + x264_preset="ultrafast", + ) + streams = json.loads( + subprocess.run( + [ + "ffprobe", + "-v", + "error", + "-count_frames", + "-show_streams", + "-of", + "json", + str(saved), + ], + capture_output=True, + text=True, + check=True, + ).stdout + )["streams"] + video = next(stream for stream in streams if stream["codec_type"] == "video") + assert (video["width"], video["height"]) == (48, 32) + assert video["codec_name"] == "h264" + assert int(video["nb_read_frames"]) == FRAMES + assert video["r_frame_rate"] == f"{FPS}/1" + options = _x264_options(saved) + if compression == 100: + assert options["qp"] == "0" + else: + assert options["crf"] == f"{int((1 - compression / 100) * 51):.1f}" + audio = [stream for stream in streams if stream["codec_type"] == "audio"] + assert bool(audio) == with_audio + if with_audio: + assert audio[0]["codec_name"] == "aac" + assert audio[0]["sample_rate"] == "32000" + assert audio[0]["channels"] == 2 + + +@pytest.mark.skipif(shutil.which("ffmpeg") is None, reason="needs ffmpeg") +def test_ffmpeg_failure_is_reported(tmp_path): + with pytest.raises(RuntimeError, match="ffmpeg video encoding failed"): + output_utils._save_video_ffmpeg( + str(tmp_path / "invalid.mp4"), + [np.zeros((16, 16, 3), dtype=np.uint8)], + fps=FPS, + quality=5, + x264_preset="invalid", + ) diff --git a/python/sglang/multimodal_gen/test/unit/test_vsa_h3_attention.py b/python/sglang/multimodal_gen/test/unit/test_vsa_h3_attention.py index 999eb2daff..09513447e3 100644 --- a/python/sglang/multimodal_gen/test/unit/test_vsa_h3_attention.py +++ b/python/sglang/multimodal_gen/test/unit/test_vsa_h3_attention.py @@ -19,6 +19,10 @@ from sglang.multimodal_gen.runtime.layers.attention.backends.video_sparse_attn_h VideoSparseAttentionH3Impl, VideoSparseAttentionH3MetadataBuilder, _topk_tile_lists, + vsa_h3_fold_gate, +) +from sglang.multimodal_gen.runtime.layers.attention.backends.vsa_h3_kernels import ( + vsa_h3_topk_lists, ) requires_cuda = pytest.mark.skipif( @@ -163,6 +167,29 @@ def test_topk_tile_list_semantics() -> None: assert (_lists_to_mask(compete, n_tiles).sum(dim=-1) == keep + num_prefix).all() +@requires_cuda +@pytest.mark.parametrize( + "num_prefix, num_video, sparsity", + [(29, 528, 0.8), (0, 64, 0.9), (30, 1541, 0.05)], +) +def test_topk_list_kernel_matches_sorted_topk(num_prefix, num_video, sparsity) -> None: + """The unsorted-top-k list kernel must emit the sorted lists and leave its + scratch mask zeroed for the next layer.""" + n_tiles = num_prefix + num_video + scores = torch.randn(HEADS, n_tiles, n_tiles, device="cuda") + reference = _topk_tile_lists(scores, num_prefix, num_video, sparsity, True) + keep = reference.shape[-1] - num_prefix + out = torch.full((HEADS, num_video, n_tiles), -1, dtype=torch.int32, device="cuda") + out[..., :num_prefix] = torch.arange(num_prefix, dtype=torch.int32, device="cuda") + mask = torch.zeros(HEADS, num_video, num_video, dtype=torch.int8, device="cuda") + for _ in range(2): # the scratch mask must come back zeroed + picked = scores[:, num_prefix:, num_prefix:].topk(keep, sorted=False).indices + vsa_h3_topk_lists(picked, mask, out, num_prefix) + assert torch.equal(out[..., : num_prefix + keep], reference) + assert not mask.any() + assert (out[..., num_prefix + keep :] == -1).all() + + def _masked_dense_reference(meta, used, q, k, v, gate, sparsity): """fp32 reference over the padded tile layout with the top-k tile mask.""" n_tiles = meta.num_tiles @@ -215,6 +242,68 @@ def test_sparse_gated_matches_masked_dense_reference() -> None: assert torch.all(out[used:] == 0) +@requires_cuda +def test_return_compress_fold_on_row_shards_is_bit_identical() -> None: + """The deferred row-shard fold (Ulysses) must reproduce the in-kernel fold + bit for bit: same tile index derivation, same fp32 FMA order.""" + device = torch.device("cuda") + meta = _build_metadata(0.5, device) + used, total, (q, k, v) = _packed_qkv(device) + gate = (torch.randn_like(q) * 0.1).to(torch.bfloat16) + impl = _impl() + + folded = _run(impl, meta, used, total, q, k, v, gate=gate) + cu = torch.tensor([0, used, total], dtype=torch.int32, device=device) + out, out_compress = impl.forward_varlen( + q, + k, + v, + cu_seqlens=cu, + max_seqlen=used, + cu_seqlens_host=(0, used, total), + attn_metadata=meta, + return_compress=True, + ) + shard = total // 2 + for row_start in (0, shard): + rows = slice(row_start, row_start + shard) + vsa_h3_fold_gate(out[rows], gate[rows], out_compress, meta, row_start) + assert torch.equal(out, folded) + + +@requires_cuda +@pytest.mark.parametrize("sparsity", [0.0, 0.8]) +def test_odd_tile_count_matches_reference(sparsity: float) -> None: + """An odd tile count (padded to even for the SM100 kernel) must keep the + pooled means, the top-k lists and the gate fold on the real tiles.""" + device = torch.device("cuda") + prefix = (70, 0, 40) + meta = VideoSparseAttentionH3MetadataBuilder().build( + current_timestep=0, + raw_latent_shape=VIDEO_SHAPE, + patch_size=(1, 1, 1), + VSA_sparsity=sparsity, + prefix_segments=prefix, + device=device, + ) + assert meta.num_tiles % 2 == 1 + used = sum(prefix) + math.prod(VIDEO_SHAPE) + total = (used + 63) // 64 * 64 + generator = torch.Generator(device="cpu").manual_seed(11) + q, k, v, gate = ( + torch.randn((total, HEADS, HEAD_DIM), generator=generator).to( + device=device, dtype=torch.bfloat16 + ) + for _ in range(4) + ) + gate = gate * 0.1 + out = _run(_impl(), meta, used, total, q, k, v, gate=gate) + reference = _masked_dense_reference(meta, used, q, k, v, gate, sparsity) + diff = (out[:used].float() - reference).abs().max().item() + assert diff < 2e-2, f"odd tile count, sparsity {sparsity}: max diff {diff}" + assert torch.all(out[used:] == 0) + + def test_metadata_tile_geometry_accounts_every_row() -> None: device = torch.device("cpu") meta = _build_metadata(0.9, device) diff --git a/test/registered/kernels/ops/diffusion/test_vsa_block_sparse_sm100.py b/test/registered/kernels/ops/diffusion/test_vsa_block_sparse_sm100.py new file mode 100644 index 0000000000..5ff1c89f11 --- /dev/null +++ b/test/registered/kernels/ops/diffusion/test_vsa_block_sparse_sm100.py @@ -0,0 +1,105 @@ +import math +import sys + +import pytest +import torch + +from sglang.kernels.ops.diffusion import ( + can_use_vsa_block_sparse_sm100, + vsa_block_sparse_sm100, +) +from sglang.test.ci.ci_register import register_cuda_ci + +register_cuda_ci(est_time=60, stage="base-b-kernel-unit", runner_config="4-gpu-b200") + +pytestmark = pytest.mark.skipif( + not torch.cuda.is_available() + or not can_use_vsa_block_sparse_sm100( + torch.cuda.current_device(), torch.bfloat16, 128 + ), + reason="needs the sm_100a / sm_103a block-sparse kernel", +) + +BLOCK = 64 + + +def _reference(q, k, v, q2k_idx, q2k_num, block_sizes, sm_scale): + b, h, s, d = q.shape + n = block_sizes.numel() + key_valid = torch.arange(BLOCK, device=q.device)[None, :] < block_sizes[:, None] + out = torch.zeros(b, h, s, d, dtype=torch.float32, device=q.device) + for bi in range(b): + for hi in range(h): + for qi in range(n): + row = (bi * h + hi) * n + qi + count = int(q2k_num[row]) + if count == 0: + continue + tiles = q2k_idx[row, :count].long() + keys = torch.cat( + [k[bi, hi, t * BLOCK : (t + 1) * BLOCK] for t in tiles] + ).float() + vals = torch.cat( + [v[bi, hi, t * BLOCK : (t + 1) * BLOCK] for t in tiles] + ).float() + mask = key_valid[tiles].reshape(-1) + scores = ( + q[bi, hi, qi * BLOCK : (qi + 1) * BLOCK].float() @ keys.T * sm_scale + ) + scores = scores.masked_fill(~mask[None, :], float("-inf")) + out[bi, hi, qi * BLOCK : (qi + 1) * BLOCK] = ( + torch.softmax(scores, -1) @ vals + ) + return out + + +@pytest.mark.parametrize("batch,heads,num_blocks,max_kv", [(1, 2, 6, 4), (2, 3, 10, 7)]) +def test_matches_masked_reference(batch, heads, num_blocks, max_kv): + """Empty lists, zero-size and ragged tiles must match the masked fp32 reference.""" + generator = torch.Generator(device="cuda").manual_seed(3) + d, s = 128, num_blocks * BLOCK + q, k, v = ( + torch.randn( + batch, heads, s, d, device="cuda", dtype=torch.bfloat16, generator=generator + ) + for _ in range(3) + ) + block_sizes = torch.randint( + 1, BLOCK + 1, (num_blocks,), device="cuda", generator=generator + ).int() + block_sizes[-1] = 0 + rows = batch * heads * num_blocks + q2k_idx = torch.stack( + [ + torch.randperm(num_blocks, device="cuda", generator=generator)[:max_kv] + .sort() + .values + for _ in range(rows) + ] + ).int() + q2k_num = torch.randint( + 0, max_kv + 1, (rows,), device="cuda", generator=generator + ).int() + q2k_num[0] = 0 + q2k_num[-1] = max_kv + sm_scale = 1.0 / math.sqrt(d) + + out = torch.empty_like(q) + vsa_block_sparse_sm100(q, k, v, q2k_idx, q2k_num, block_sizes, out, sm_scale) + reference = _reference(q, k, v, q2k_idx, q2k_num, block_sizes, sm_scale) + torch.testing.assert_close(out.float(), reference, atol=2e-2, rtol=2e-2) + assert torch.all(out[0, 0, :BLOCK] == 0) + + +def test_rejects_odd_tile_count(): + """An odd tile count must raise instead of reading past the paired query tile.""" + q = torch.randn(1, 1, 3 * BLOCK, 128, device="cuda", dtype=torch.bfloat16) + idx = torch.zeros(3, 1, device="cuda", dtype=torch.int32) + num = torch.ones(3, device="cuda", dtype=torch.int32) + sizes = torch.full((3,), BLOCK, device="cuda", dtype=torch.int32) + with pytest.raises(RuntimeError, match="even"): + vsa_block_sparse_sm100(q, q, q, idx, num, sizes, torch.empty_like(q), 0.1) + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-v"]))