From ccae3b36393ea7d063d2838ab16b2b11f4da0b56 Mon Sep 17 00:00:00 2001 From: Mick Date: Sat, 10 Oct 2026 12:58:30 +0800 Subject: [PATCH] [diffusion] perf: pipeline MiniMax-H3's Ulysses exchange against cuDNN SDPA on Blackwell (#43373) Co-authored-by: Mick Qian Co-authored-by: Claude Opus 5.5 --- docs/cookbook/diffusion/MiniMax/MiniMax-H3.mdx | 2 +- .../runtime/models/dits/minimax_h3.py | 18 ++++++++++++++---- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/docs/cookbook/diffusion/MiniMax/MiniMax-H3.mdx b/docs/cookbook/diffusion/MiniMax/MiniMax-H3.mdx index 1e1df5c250..90442bd517 100644 --- a/docs/cookbook/diffusion/MiniMax/MiniMax-H3.mdx +++ b/docs/cookbook/diffusion/MiniMax/MiniMax-H3.mdx @@ -1475,7 +1475,7 @@ fold decision is not node-boundary aware: - The released visual VAE quality recipe uses overlapping tiled decode. SGLang keeps that recipe by default and distributes complete tiles across the decode group; this changes scheduling, not the computation inside each tile. - H3 rejects `--vae-config.parallel-decode-mode spatial` and `spatial_shard`: validation found output mismatches. Use the default released tiled recipe. - Keep the default `--encoder-parallel auto`. With the server’s default `batching_max_size` of 1, single-node H100/H200/B200/B300 recipes with peer-to-peer access fold the Qwen text encoder over otherwise idle Ulysses ranks. This is separate from DiT tensor parallelism. A pure-TP recipe already shards the encoder over its TP group and does not add a world fold. -- Single-node Ulysses recipes pipeline each block's all-to-all against dense FlashAttention over the copy engine: one head group's q/k/v moves while the previous group attends, every copy (the merge into the output included) runs on the copy engine, and the output is bit-identical to the sequential exchange. It needs peer-to-peer access and stream memory operations on every rank and about 10 × (rows per rank) × heads × head_dim extra activations per rank (3.6 GB for Ref2VA at Ulysses 2); when that does not fit on some rank, every rank keeps the sequential exchange. `SGLANG_DIFFUSION_ULYSSES_PIPELINE_GROUPS=0` restores the sequential exchange. +- Single-node Ulysses recipes pipeline each block's all-to-all against dense attention (FlashAttention on Hopper, cuDNN SDPA on Blackwell) over the copy engine: one head group's q/k/v moves while the previous group attends, every copy (the merge into the output included) runs on the copy engine, and the output is bit-identical to the sequential exchange. It needs peer-to-peer access and stream memory operations on every rank and about 10 × (rows per rank) × heads × head_dim extra activations per rank (3.6 GB for Ref2VA at Ulysses 2); when that does not fit on some rank, every rank keeps the sequential exchange. `SGLANG_DIFFUSION_ULYSSES_PIPELINE_GROUPS=0` restores the sequential exchange. - For throughput-oriented serving, select **DP (batched throughput)**. The picker pairs `--encoder-parallel dp` with an editable `--batching-max-size` greater than 1. Encoder DP stays inside each DiT replica and composes with encoder TP: the H100 TP2 + Ulysses2 recipe has two TP-sharded encoder copies that can split a batch, while the RTX 5090 pure-TP2 recipe has one encoder copy and therefore no additional batch-DP degree. It provides no benefit for a batch of one and is not bitwise-identical to the unsplit deployment. - Use explicit **Fold** to prioritize single-request latency and encoder memory on a measured high-bandwidth single-node topology. Use **Replicate** as the compatibility path when folding or encoder DP is unsuitable. - `--use-fsdp-inference true` shards only the DiT. MiniMax-H3 preserves the original FP32 dtype of its patch, time, and output projections during FSDP all-gather, so this path does not trade numerical correctness for memory. On 4×H100, prefer TP2 + Ulysses2 for speed; use FSDP as an explicit capacity policy rather than assuming it is faster. diff --git a/python/sglang/multimodal_gen/runtime/models/dits/minimax_h3.py b/python/sglang/multimodal_gen/runtime/models/dits/minimax_h3.py index bc41a2904b..970cdc63ea 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/minimax_h3.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/minimax_h3.py @@ -682,6 +682,14 @@ def _minimax_h3_bind_attention_backend( ) +# FlashAttention on Hopper and cuDNN SDPA on Blackwell give a head group the +# same bytes as those heads inside the full call (checked at H3's shapes) +_PIPELINED_ATTENTION_BACKENDS = ( + AttentionBackendEnum.FA, + AttentionBackendEnum.DYNAMIC_CUDNN_SDPA, +) + + def _minimax_h3_pipelined_dense_attention( attention: MiniMaxH3Attention, q: torch.Tensor, @@ -693,18 +701,20 @@ def _minimax_h3_pipelined_dense_attention( max_seqlen: int, fill=None, ) -> torch.Tensor | None: - """Dense FA with the Ulysses exchange pipelined over head groups. + """Dense attention with the Ulysses exchange pipelined over head groups. Returns None (caller keeps the sequential exchange) unless head-group - pipelining is enabled and the dense FlashAttention path would run. + pipelining is enabled and a dense backend that runs a head group exactly + as it runs those heads in the full call would serve it. """ groups = envs.SGLANG_DIFFUSION_ULYSSES_PIPELINE_GROUPS if groups in (0, 1) or torch.compiler.is_compiling(): return None - if attention._attention_backend_enum is not AttentionBackendEnum.FA: + if attention._attention_backend_enum not in _PIPELINED_ATTENTION_BACKENDS: return None impl = attention._attention_impl - if impl._request_skip_softmax_threshold()[0]: + skip_softmax = getattr(impl, "_request_skip_softmax_threshold", None) + if skip_softmax is not None and skip_softmax()[0]: return None from sglang.multimodal_gen.runtime.distributed.device_communicators.ipc_a2a_multi import ( ulysses_pipelined_attention,