From 85374532f265d65ccb276f2f99ced57520acab4f Mon Sep 17 00:00:00 2001 From: Xiaoyu Zhang <1182563586@qq.com> Date: Sat, 26 Sep 2026 09:24:21 +0800 Subject: [PATCH] [diffusion] Fuse lossless Klein packed QK RMSNorm and RoPE on Hopper (#40490) --- docs/docs/sglang-diffusion/fused_kernels.mdx | 8 ++ .../sglang/kernels/ops/diffusion/__init__.py | 2 + .../rope/flux2_qknorm_rope_triton.py | 130 ++++++++++++++++++ .../runtime/models/dits/flux_2.py | 88 ++++++++++-- .../test_flux2_strided_qknorm_rope.py | 117 ++++++++++++++++ 5 files changed, 332 insertions(+), 13 deletions(-) create mode 100644 python/sglang/kernels/ops/diffusion/rope/flux2_qknorm_rope_triton.py create mode 100644 test/registered/kernels/ops/diffusion/test_flux2_strided_qknorm_rope.py diff --git a/docs/docs/sglang-diffusion/fused_kernels.mdx b/docs/docs/sglang-diffusion/fused_kernels.mdx index 9ff506a7c4..138ed98ded 100644 --- a/docs/docs/sglang-diffusion/fused_kernels.mdx +++ b/docs/docs/sglang-diffusion/fused_kernels.mdx @@ -142,6 +142,7 @@ kernel replay support does not enable BCG for a model configuration that disable | --- | --- | --- | --- | | `fused_inplace_qknorm_rope` | JIT CUDA | one bf16 rounding step vs the split baseline; exact with `round_norm_before_rope=True` | separate QK-norm kernel + RoPE | | `flux2_qkv_epilogue` | KDA (JIT CUDA) | bit-exact against its selected BF16 reference chain | FLUX.2 QK RMSNorm + RoPE + joint text/image QKV packing | +| `flux2_strided_qknorm_rope` | Triton | bit-exact against native tiled RMSNorm followed by FlashInfer interleaved RoPE; verified per signature | FLUX.2 Klein single-stream packed Q/K copies + QK RMSNorm + RoPE on Hopper, BF16, 24 heads of dimension 128 | | `qwen_qkv_epilogue` | JIT CUDA | bit-exact against its selected BF16 reference chain | Qwen-Image QK RMSNorm + RoPE + joint QKV writes on SM100+ | | `rope_rotate_half` | Triton | bit-exact | `chunk` → `cat(-x2, x1)` → two muls + add → `cat(tail)`, about 7 kernels per projection | | `interleaved_rope_fp64` | JIT CUDA | bit-exact | paired SANA-Video Q/K RoPE with fp64 tables, about 14 eager kernels | @@ -150,6 +151,13 @@ kernel replay support does not enable BCG for a model configuration that disable | `ltx25_decoder_rope` | JIT CUDA | bit-exact | paired LTX-2.5 decoder 3D RoPE from cached compact axis tables | | `hunyuan_qkv_rope_pack` | Triton | bit-exact | QKV pack and RoPE in one pass | +The Klein strided path preserves the BF16 rounding between RMSNorm and RoPE, +including signed zeros. It reads packed Q/K views without changing the adjacent +V or MLP values. Unsupported layouts, devices, or RMSNorm dispatches use the +original helper. A new signature is verified outside CUDA Graph capture; an +unverified capture falls back. `SGLANG_ENABLE_FUSED_QKNORM_ROPE=0` also disables +this path. This kernel does not enable model-level BCG support for FLUX.2. + ### Activation | Operator | Backend | Contract | Replaces | diff --git a/python/sglang/kernels/ops/diffusion/__init__.py b/python/sglang/kernels/ops/diffusion/__init__.py index fcd20a1410..351b9c426d 100644 --- a/python/sglang/kernels/ops/diffusion/__init__.py +++ b/python/sglang/kernels/ops/diffusion/__init__.py @@ -573,6 +573,8 @@ _EXPORTS: dict[str, str] = { "try_fused_bias_mul_add": "sglang.kernels.kda_kernels.norm_scale_shift_jit", "try_fused_bias_scale_residual_norm_scale_shift": "sglang.kernels.kda_kernels.norm_scale_shift_jit", "triton_one_pass_rms_norm": "norm.rmsnorm_onepass_triton", + "can_use_flux2_strided_qknorm_rope": "rope.flux2_qknorm_rope_triton", + "flux2_strided_qknorm_rope": "rope.flux2_qknorm_rope_triton", "can_use_rmsnorm_preserve_reduction": "norm.rmsnorm_preserve_reduction", "rmsnorm_preserve_reduction": "norm.rmsnorm_preserve_reduction", "can_use_fused_rmsnorm_scale_shift": "norm.rmsnorm_scale_shift_bitexact", diff --git a/python/sglang/kernels/ops/diffusion/rope/flux2_qknorm_rope_triton.py b/python/sglang/kernels/ops/diffusion/rope/flux2_qknorm_rope_triton.py new file mode 100644 index 0000000000..a1d6de050d --- /dev/null +++ b/python/sglang/kernels/ops/diffusion/rope/flux2_qknorm_rope_triton.py @@ -0,0 +1,130 @@ +"""Packed FLUX.2 Q/K RMSNorm and interleaved RoPE with native rounding.""" + +import torch +import triton +import triton.language as tl + + +def can_use_flux2_strided_qknorm_rope(q, k, q_weight, k_weight, cache) -> bool: + if ( + torch.compiler.is_compiling() + or torch.is_grad_enabled() + or torch.version.hip is not None + or not q.is_cuda + or torch.cuda.get_device_capability(q.device) != (9, 0) + or q.ndim != 4 + or q.shape[-2:] != (24, 128) + or q.numel() == 0 + or q.is_contiguous() + or k.shape != q.shape + or k.stride() != q.stride() + or q.stride(-1) != 1 + or q.stride(-2) != 128 + or q.stride(0) != q.shape[1] * q.stride(1) + or q.stride(1) < 24 * 128 + ): + return False + if any( + t.device != q.device or t.dtype != torch.bfloat16 + for t in (q, k, q_weight, k_weight) + ): + return False + if any(t.shape != (128,) or not t.is_contiguous() for t in (q_weight, k_weight)): + return False + return ( + isinstance(cache, torch.Tensor) + and cache.device == q.device + and cache.dtype == torch.float32 + and cache.ndim == 2 + and cache.shape[0] >= q.shape[1] + and cache.shape[1] == 128 + and cache.is_contiguous() + ) + + +@triton.jit +def _flux2_strided_qknorm_rope_kernel( + q_ptr, + k_ptr, + qw_ptr, + kw_ptr, + cache_ptr, + qo_ptr, + ko_ptr, + ROWS: tl.constexpr, + TOKENS: tl.constexpr, + HEADS: tl.constexpr, + TOKEN_STRIDE: tl.constexpr, + CACHE_STRIDE: tl.constexpr, + EPS: tl.constexpr, + BLOCK_ROWS: tl.constexpr, +): + rows = tl.program_id(0) * BLOCK_ROWS + tl.arange(0, BLOCK_ROWS)[:, None] + dims = tl.arange(0, 128)[None, :] + if tl.program_id(1) == 0: + x_ptr, w_ptr, y_ptr = q_ptr, qw_ptr, qo_ptr + else: + x_ptr, w_ptr, y_ptr = k_ptr, kw_ptr, ko_ptr + offset = (rows // HEADS).to(tl.int64) * TOKEN_STRIDE + rows % HEADS * 128 + x = tl.load(x_ptr + offset + dims, mask=rows < ROWS, other=0.0).to(tl.float32) + # Keep the exact arithmetic and row tiling of _rms_norm_tiled_onepass. + mean_square = tl.sum(x * x, axis=1, keep_dims=True) / 128 + rstd = tl.math.rsqrt(mean_square + EPS) + w = tl.load(w_ptr + dims) + normalized = (x * rstd * w).to(y_ptr.dtype.element_ty).to(tl.float32) + even, odd = tl.split(tl.reshape(normalized, (BLOCK_ROWS, 64, 2))) + pos = rows // HEADS % TOKENS + half = tl.arange(0, 64)[None, :] + cos = tl.load(cache_ptr + pos * CACHE_STRIDE + half, mask=rows < ROWS, other=0.0) + sin = tl.load( + cache_ptr + pos * CACHE_STRIDE + 64 + half, mask=rows < ROWS, other=0.0 + ) + # FlashInfer rounds the sine product before the cosine multiply-add. + # A subtraction from +0 can canonicalize signed zeros. Flip the IEEE + # sign bit explicitly, matching FlashInfer's even-lane negation. + negative_odd = (odd.to(tl.int32, bitcast=True) ^ -2147483648).to( + tl.float32, bitcast=True + ) + rotated_even = tl.fma(even, cos, negative_odd * sin) + rotated_odd = tl.fma(odd, cos, even * sin) + rotated = tl.reshape(tl.join(rotated_even, rotated_odd), (BLOCK_ROWS, 128)) + tl.store(y_ptr + rows * 128 + dims, rotated, mask=rows < ROWS) + + +def flux2_strided_qknorm_rope( + q: torch.Tensor, + k: torch.Tensor, + q_weight: torch.Tensor, + k_weight: torch.Tensor, + cos_sin_cache: torch.Tensor, + eps: float, +) -> tuple[torch.Tensor, torch.Tensor]: + batch, tokens, heads, dim = q.shape + assert dim == 128 and k.shape == q.shape + assert q.stride() == k.stride() + assert q.stride(-1) == 1 and q.stride(-2) == 128 + assert q.stride(0) == tokens * q.stride(1) + assert q.dtype == k.dtype == q_weight.dtype == k_weight.dtype == torch.bfloat16 + assert cos_sin_cache.dtype == torch.float32 and cos_sin_cache.shape[1] == 128 + output_q = torch.empty(q.shape, device=q.device, dtype=q.dtype) + output_k = torch.empty_like(output_q) + rows = batch * tokens * heads + block_rows = min(16, triton.next_power_of_2(max(1, rows // 512))) + with torch.cuda.device(q.device): + _flux2_strided_qknorm_rope_kernel[(triton.cdiv(rows, block_rows), 2)]( + q, + k, + q_weight, + k_weight, + cos_sin_cache, + output_q, + output_k, + rows, + tokens, + heads, + q.stride(1), + cos_sin_cache.stride(0), + eps, + BLOCK_ROWS=block_rows, + ) + return output_q, output_k diff --git a/python/sglang/multimodal_gen/runtime/models/dits/flux_2.py b/python/sglang/multimodal_gen/runtime/models/dits/flux_2.py index 6203f58ba4..d5854576cc 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/flux_2.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/flux_2.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os from typing import Any, Dict, List, Optional, Tuple import torch @@ -25,9 +26,11 @@ from sglang.kernels.ops.diffusion import ( BitExactFusionGate, can_defer_flux2_gated_residual, can_use_flux2_gated_resnorm, + can_use_flux2_strided_qknorm_rope, can_use_fused_layernorm_modulate, flux2_gated_resnorm_raw, flux2_nvfp4_swiglu_quant_active, + flux2_strided_qknorm_rope, fused_layernorm_modulate_fp8_quant_raw, fused_layernorm_modulate_raw, fused_packed_silu_mul_bitexact, @@ -103,6 +106,68 @@ _FLUX2_LN_FP8 = BitExactFusionGate( ) _FLUX2_LN_FP8_SIGS = _FLUX2_LN_FP8.verified_sigs assert _FLUX2_LN_FP8_SIGS is not None +_FLUX2_STRIDED_QK_ROPE = BitExactFusionGate( + "FLUX.2 Klein strided QK RMSNorm+RoPE", per_signature=True +) + + +def _flux2_single_qk_rope(q, k, q_norm, k_norm, head_dim, cache, complex_freqs): + def reference(): + return apply_qk_norm_with_optional_rope( + q=q, + k=k, + q_norm=q_norm, + k_norm=k_norm, + head_dim=head_dim, + cos_sin_cache=cache, + freqs_complex=complex_freqs, + is_neox=False, + allow_inplace=True, + allow_strided_qk=current_platform.is_rocm(), + ) + + gate = _FLUX2_STRIDED_QK_ROPE + if ( + gate.disabled + or head_dim != 128 + or type(q_norm) is not RMSNorm + or type(k_norm) is not RMSNorm + or q_norm.variance_epsilon != k_norm.variance_epsilon + or any( + norm.variance_size_override is not None + or getattr(norm._forward_method, "__func__", None) + is not RMSNorm.forward_cuda + for norm in (q_norm, k_norm) + ) + or os.getenv("SGLANG_ENABLE_FUSED_QKNORM_ROPE", "1").lower() + in {"0", "false", "off", "no"} + or not can_use_flux2_strided_qknorm_rope( + q, k, q_norm.weight, k_norm.weight, cache + ) + ): + return reference() + sig = (q.device, q.shape, q.stride(), cache.stride(), q_norm.variance_epsilon) + verified = gate.is_verified(sig) + if not verified and torch.cuda.is_current_stream_capturing(): + return reference() + try: + out = flux2_strided_qknorm_rope( + q, k, q_norm.weight, k_norm.weight, cache, q_norm.variance_epsilon + ) + except Exception as exc: + gate.on_exception(exc, logger=logger) + return reference() + if verified: + return out + return gate.accept_or_fallback( + out, + reference(), + sig=sig, + equal=lambda a, b: all( + torch.equal(x.view(torch.int16), y.view(torch.int16)) for x, y in zip(a, b) + ), + logger=logger, + ) def _valid_modelopt_fp8_linear(linear: nn.Module) -> bool: @@ -948,19 +1013,16 @@ class Flux2ParallelSelfAttention(torch.nn.Module, AttentionModuleMixin): if complex_freqs is not None: complex_freqs = complex_freqs[: query.shape[1]] - # QK-norm (+ RoPE) via the shared helper so the fused kernel path is used - # here too — the single-stream block previously ran norm and RoPE as separate ops. - query, key = apply_qk_norm_with_optional_rope( - q=query, - k=key, - q_norm=self.norm_q, - k_norm=self.norm_k, - head_dim=self.head_dim, - cos_sin_cache=cos_sin_cache, - freqs_complex=complex_freqs, - is_neox=False, - allow_inplace=True, - allow_strided_qk=True if current_platform.is_rocm() else False, + # Packed QKV/MLP projection views can be normalized and rotated + # without materializing contiguous Q/K inputs on the supported path. + query, key = _flux2_single_qk_rope( + query, + key, + self.norm_q, + self.norm_k, + self.head_dim, + cos_sin_cache, + complex_freqs, ) hidden_states = self.attn( query, diff --git a/test/registered/kernels/ops/diffusion/test_flux2_strided_qknorm_rope.py b/test/registered/kernels/ops/diffusion/test_flux2_strided_qknorm_rope.py new file mode 100644 index 0000000000..40ccd14fe4 --- /dev/null +++ b/test/registered/kernels/ops/diffusion/test_flux2_strided_qknorm_rope.py @@ -0,0 +1,117 @@ +"""Klein packed Q/K fusion against the actual native RMSNorm/RoPE dispatch.""" + +import unittest +from unittest.mock import patch + +import torch + +from sglang.kernels.ops.diffusion import ( + BitExactFusionGate, + can_use_flux2_strided_qknorm_rope, + flux2_strided_qknorm_rope, +) +from sglang.multimodal_gen.runtime.layers.layernorm import ( + RMSNorm, + apply_qk_norm_with_optional_rope, +) +from sglang.multimodal_gen.runtime.models.dits import flux_2 +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=25, stage="base-b-kernel-unit", runner_config="1-gpu-large") + + +def inputs(batch, tokens): + packed = torch.randn(batch, tokens, 9 * 3072, device="cuda", dtype=torch.bfloat16) + q, k, _ = packed[:, :, : 3 * 3072].chunk(3, dim=-1) + q, k = (x.unflatten(-1, (24, 128)) for x in (q, k)) + qn, kn = (RMSNorm(128, eps=1e-6).to("cuda", torch.bfloat16) for _ in range(2)) + qn.weight.normal_() + kn.weight.normal_() + angles = torch.randn(tokens, 64, device="cuda") + cache = torch.cat((angles.cos(), angles.sin()), dim=-1) + return packed, q, k, qn, kn, cache + + +def reference(q, k, qn, kn, cache): + return apply_qk_norm_with_optional_rope( + q, + k, + qn, + kn, + 128, + cache, + is_neox=False, + allow_inplace=True, + allow_strided_qk=False, + ) + + +@unittest.skipUnless( + torch.cuda.is_available() and torch.version.hip is None, "NVIDIA CUDA required" +) +class TestFlux2StridedQKNormRoPE(CustomTestCase): + def assert_bits(self, actual, expected): + for a, b in zip(actual, expected, strict=True): + self.assertEqual(a.dtype, b.dtype) + self.assertTrue(torch.equal(a.view(torch.int16), b.view(torch.int16))) + + @torch.inference_mode() + def test_native_shapes_magnitudes_and_untouched_packed_input(self): + torch.manual_seed(42) + for batch, tokens in [(1, 17), (2, 257), (1, 4608)]: + packed, q, k, qn, kn, cache = inputs(batch, tokens) + for magnitude in (0.005, 1.0, 200.0): + with self.subTest(batch=batch, tokens=tokens, magnitude=magnitude): + packed.normal_().mul_(magnitude) + original = packed.clone() + gate = BitExactFusionGate("test Klein QK", per_signature=True) + with patch.object(flux_2, "_FLUX2_STRIDED_QK_ROPE", gate): + actual = flux_2._flux2_single_qk_rope( + q, k, qn, kn, 128, cache, None + ) + self.assert_bits(actual, reference(q, k, qn, kn, cache)) + self.assertTrue( + torch.equal( + packed.view(torch.int16), original.view(torch.int16) + ) + ) + if can_use_flux2_strided_qknorm_rope( + q, k, qn.weight, kn.weight, cache + ): + self.assertTrue(gate.verified) + self.assertFalse(gate.disabled) + self.assert_bits( + flux2_strided_qknorm_rope( + q, k, qn.weight, kn.weight, cache, qn.variance_epsilon + ), + reference(q, k, qn, kn, cache), + ) + else: + self.assertFalse(gate.verified) + self.assertFalse(gate.disabled) + + @torch.inference_mode() + def test_changed_inputs_weights_cache_and_graph_replay(self): + packed, q, k, qn, kn, cache = inputs(1, 513) + gate = BitExactFusionGate("test Klein replay", per_signature=True) + with patch.object(flux_2, "_FLUX2_STRIDED_QK_ROPE", gate): + flux_2._flux2_single_qk_rope(q, k, qn, kn, 128, cache, None) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + out = flux_2._flux2_single_qk_rope(q, k, qn, kn, 128, cache, None) + for seed in range(8): + torch.manual_seed(seed) + packed.normal_() + qn.weight.normal_() + kn.weight.normal_() + cache.normal_() + graph.replay() + self.assert_bits(out, reference(q, k, qn, kn, cache)) + packed.zero_().neg_() + graph.replay() + self.assert_bits(out, reference(q, k, qn, kn, cache)) + + +if __name__ == "__main__": + unittest.main()