[diffusion] Fuse lossless Klein packed QK RMSNorm and RoPE on Hopper (#40490)

This commit is contained in:
Xiaoyu Zhang
2026-09-26 09:24:21 +08:00
committed by GitHub
parent c842d32b0a
commit 85374532f2
5 changed files with 332 additions and 13 deletions
@@ -573,6 +573,8 @@ _EXPORTS: dict[str, str] = {
"try_fused_bias_mul_add": "sglang.kernels.kda_kernels.norm_scale_shift_jit",
"try_fused_bias_scale_residual_norm_scale_shift": "sglang.kernels.kda_kernels.norm_scale_shift_jit",
"triton_one_pass_rms_norm": "norm.rmsnorm_onepass_triton",
"can_use_flux2_strided_qknorm_rope": "rope.flux2_qknorm_rope_triton",
"flux2_strided_qknorm_rope": "rope.flux2_qknorm_rope_triton",
"can_use_rmsnorm_preserve_reduction": "norm.rmsnorm_preserve_reduction",
"rmsnorm_preserve_reduction": "norm.rmsnorm_preserve_reduction",
"can_use_fused_rmsnorm_scale_shift": "norm.rmsnorm_scale_shift_bitexact",
@@ -0,0 +1,130 @@
"""Packed FLUX.2 Q/K RMSNorm and interleaved RoPE with native rounding."""
import torch
import triton
import triton.language as tl
def can_use_flux2_strided_qknorm_rope(q, k, q_weight, k_weight, cache) -> bool:
if (
torch.compiler.is_compiling()
or torch.is_grad_enabled()
or torch.version.hip is not None
or not q.is_cuda
or torch.cuda.get_device_capability(q.device) != (9, 0)
or q.ndim != 4
or q.shape[-2:] != (24, 128)
or q.numel() == 0
or q.is_contiguous()
or k.shape != q.shape
or k.stride() != q.stride()
or q.stride(-1) != 1
or q.stride(-2) != 128
or q.stride(0) != q.shape[1] * q.stride(1)
or q.stride(1) < 24 * 128
):
return False
if any(
t.device != q.device or t.dtype != torch.bfloat16
for t in (q, k, q_weight, k_weight)
):
return False
if any(t.shape != (128,) or not t.is_contiguous() for t in (q_weight, k_weight)):
return False
return (
isinstance(cache, torch.Tensor)
and cache.device == q.device
and cache.dtype == torch.float32
and cache.ndim == 2
and cache.shape[0] >= q.shape[1]
and cache.shape[1] == 128
and cache.is_contiguous()
)
@triton.jit
def _flux2_strided_qknorm_rope_kernel(
q_ptr,
k_ptr,
qw_ptr,
kw_ptr,
cache_ptr,
qo_ptr,
ko_ptr,
ROWS: tl.constexpr,
TOKENS: tl.constexpr,
HEADS: tl.constexpr,
TOKEN_STRIDE: tl.constexpr,
CACHE_STRIDE: tl.constexpr,
EPS: tl.constexpr,
BLOCK_ROWS: tl.constexpr,
):
rows = tl.program_id(0) * BLOCK_ROWS + tl.arange(0, BLOCK_ROWS)[:, None]
dims = tl.arange(0, 128)[None, :]
if tl.program_id(1) == 0:
x_ptr, w_ptr, y_ptr = q_ptr, qw_ptr, qo_ptr
else:
x_ptr, w_ptr, y_ptr = k_ptr, kw_ptr, ko_ptr
offset = (rows // HEADS).to(tl.int64) * TOKEN_STRIDE + rows % HEADS * 128
x = tl.load(x_ptr + offset + dims, mask=rows < ROWS, other=0.0).to(tl.float32)
# Keep the exact arithmetic and row tiling of _rms_norm_tiled_onepass.
mean_square = tl.sum(x * x, axis=1, keep_dims=True) / 128
rstd = tl.math.rsqrt(mean_square + EPS)
w = tl.load(w_ptr + dims)
normalized = (x * rstd * w).to(y_ptr.dtype.element_ty).to(tl.float32)
even, odd = tl.split(tl.reshape(normalized, (BLOCK_ROWS, 64, 2)))
pos = rows // HEADS % TOKENS
half = tl.arange(0, 64)[None, :]
cos = tl.load(cache_ptr + pos * CACHE_STRIDE + half, mask=rows < ROWS, other=0.0)
sin = tl.load(
cache_ptr + pos * CACHE_STRIDE + 64 + half, mask=rows < ROWS, other=0.0
)
# FlashInfer rounds the sine product before the cosine multiply-add.
# A subtraction from +0 can canonicalize signed zeros. Flip the IEEE
# sign bit explicitly, matching FlashInfer's even-lane negation.
negative_odd = (odd.to(tl.int32, bitcast=True) ^ -2147483648).to(
tl.float32, bitcast=True
)
rotated_even = tl.fma(even, cos, negative_odd * sin)
rotated_odd = tl.fma(odd, cos, even * sin)
rotated = tl.reshape(tl.join(rotated_even, rotated_odd), (BLOCK_ROWS, 128))
tl.store(y_ptr + rows * 128 + dims, rotated, mask=rows < ROWS)
def flux2_strided_qknorm_rope(
q: torch.Tensor,
k: torch.Tensor,
q_weight: torch.Tensor,
k_weight: torch.Tensor,
cos_sin_cache: torch.Tensor,
eps: float,
) -> tuple[torch.Tensor, torch.Tensor]:
batch, tokens, heads, dim = q.shape
assert dim == 128 and k.shape == q.shape
assert q.stride() == k.stride()
assert q.stride(-1) == 1 and q.stride(-2) == 128
assert q.stride(0) == tokens * q.stride(1)
assert q.dtype == k.dtype == q_weight.dtype == k_weight.dtype == torch.bfloat16
assert cos_sin_cache.dtype == torch.float32 and cos_sin_cache.shape[1] == 128
output_q = torch.empty(q.shape, device=q.device, dtype=q.dtype)
output_k = torch.empty_like(output_q)
rows = batch * tokens * heads
block_rows = min(16, triton.next_power_of_2(max(1, rows // 512)))
with torch.cuda.device(q.device):
_flux2_strided_qknorm_rope_kernel[(triton.cdiv(rows, block_rows), 2)](
q,
k,
q_weight,
k_weight,
cos_sin_cache,
output_q,
output_k,
rows,
tokens,
heads,
q.stride(1),
cos_sin_cache.stride(0),
eps,
BLOCK_ROWS=block_rows,
)
return output_q, output_k
@@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import os
from typing import Any, Dict, List, Optional, Tuple
import torch
@@ -25,9 +26,11 @@ from sglang.kernels.ops.diffusion import (
BitExactFusionGate,
can_defer_flux2_gated_residual,
can_use_flux2_gated_resnorm,
can_use_flux2_strided_qknorm_rope,
can_use_fused_layernorm_modulate,
flux2_gated_resnorm_raw,
flux2_nvfp4_swiglu_quant_active,
flux2_strided_qknorm_rope,
fused_layernorm_modulate_fp8_quant_raw,
fused_layernorm_modulate_raw,
fused_packed_silu_mul_bitexact,
@@ -103,6 +106,68 @@ _FLUX2_LN_FP8 = BitExactFusionGate(
)
_FLUX2_LN_FP8_SIGS = _FLUX2_LN_FP8.verified_sigs
assert _FLUX2_LN_FP8_SIGS is not None
_FLUX2_STRIDED_QK_ROPE = BitExactFusionGate(
"FLUX.2 Klein strided QK RMSNorm+RoPE", per_signature=True
)
def _flux2_single_qk_rope(q, k, q_norm, k_norm, head_dim, cache, complex_freqs):
def reference():
return apply_qk_norm_with_optional_rope(
q=q,
k=k,
q_norm=q_norm,
k_norm=k_norm,
head_dim=head_dim,
cos_sin_cache=cache,
freqs_complex=complex_freqs,
is_neox=False,
allow_inplace=True,
allow_strided_qk=current_platform.is_rocm(),
)
gate = _FLUX2_STRIDED_QK_ROPE
if (
gate.disabled
or head_dim != 128
or type(q_norm) is not RMSNorm
or type(k_norm) is not RMSNorm
or q_norm.variance_epsilon != k_norm.variance_epsilon
or any(
norm.variance_size_override is not None
or getattr(norm._forward_method, "__func__", None)
is not RMSNorm.forward_cuda
for norm in (q_norm, k_norm)
)
or os.getenv("SGLANG_ENABLE_FUSED_QKNORM_ROPE", "1").lower()
in {"0", "false", "off", "no"}
or not can_use_flux2_strided_qknorm_rope(
q, k, q_norm.weight, k_norm.weight, cache
)
):
return reference()
sig = (q.device, q.shape, q.stride(), cache.stride(), q_norm.variance_epsilon)
verified = gate.is_verified(sig)
if not verified and torch.cuda.is_current_stream_capturing():
return reference()
try:
out = flux2_strided_qknorm_rope(
q, k, q_norm.weight, k_norm.weight, cache, q_norm.variance_epsilon
)
except Exception as exc:
gate.on_exception(exc, logger=logger)
return reference()
if verified:
return out
return gate.accept_or_fallback(
out,
reference(),
sig=sig,
equal=lambda a, b: all(
torch.equal(x.view(torch.int16), y.view(torch.int16)) for x, y in zip(a, b)
),
logger=logger,
)
def _valid_modelopt_fp8_linear(linear: nn.Module) -> bool:
@@ -948,19 +1013,16 @@ class Flux2ParallelSelfAttention(torch.nn.Module, AttentionModuleMixin):
if complex_freqs is not None:
complex_freqs = complex_freqs[: query.shape[1]]
# QK-norm (+ RoPE) via the shared helper so the fused kernel path is used
# here too — the single-stream block previously ran norm and RoPE as separate ops.
query, key = apply_qk_norm_with_optional_rope(
q=query,
k=key,
q_norm=self.norm_q,
k_norm=self.norm_k,
head_dim=self.head_dim,
cos_sin_cache=cos_sin_cache,
freqs_complex=complex_freqs,
is_neox=False,
allow_inplace=True,
allow_strided_qk=True if current_platform.is_rocm() else False,
# Packed QKV/MLP projection views can be normalized and rotated
# without materializing contiguous Q/K inputs on the supported path.
query, key = _flux2_single_qk_rope(
query,
key,
self.norm_q,
self.norm_k,
self.head_dim,
cos_sin_cache,
complex_freqs,
)
hidden_states = self.attn(
query,