[diffusion] Fuse lossless Klein packed QK RMSNorm and RoPE on Hopper (#40490)
This commit is contained in:
@@ -142,6 +142,7 @@ kernel replay support does not enable BCG for a model configuration that disable
|
||||
| --- | --- | --- | --- |
|
||||
| `fused_inplace_qknorm_rope` | JIT CUDA | one bf16 rounding step vs the split baseline; exact with `round_norm_before_rope=True` | separate QK-norm kernel + RoPE |
|
||||
| `flux2_qkv_epilogue` | KDA (JIT CUDA) | bit-exact against its selected BF16 reference chain | FLUX.2 QK RMSNorm + RoPE + joint text/image QKV packing |
|
||||
| `flux2_strided_qknorm_rope` | Triton | bit-exact against native tiled RMSNorm followed by FlashInfer interleaved RoPE; verified per signature | FLUX.2 Klein single-stream packed Q/K copies + QK RMSNorm + RoPE on Hopper, BF16, 24 heads of dimension 128 |
|
||||
| `qwen_qkv_epilogue` | JIT CUDA | bit-exact against its selected BF16 reference chain | Qwen-Image QK RMSNorm + RoPE + joint QKV writes on SM100+ |
|
||||
| `rope_rotate_half` | Triton | bit-exact | `chunk` → `cat(-x2, x1)` → two muls + add → `cat(tail)`, about 7 kernels per projection |
|
||||
| `interleaved_rope_fp64` | JIT CUDA | bit-exact | paired SANA-Video Q/K RoPE with fp64 tables, about 14 eager kernels |
|
||||
@@ -150,6 +151,13 @@ kernel replay support does not enable BCG for a model configuration that disable
|
||||
| `ltx25_decoder_rope` | JIT CUDA | bit-exact | paired LTX-2.5 decoder 3D RoPE from cached compact axis tables |
|
||||
| `hunyuan_qkv_rope_pack` | Triton | bit-exact | QKV pack and RoPE in one pass |
|
||||
|
||||
The Klein strided path preserves the BF16 rounding between RMSNorm and RoPE,
|
||||
including signed zeros. It reads packed Q/K views without changing the adjacent
|
||||
V or MLP values. Unsupported layouts, devices, or RMSNorm dispatches use the
|
||||
original helper. A new signature is verified outside CUDA Graph capture; an
|
||||
unverified capture falls back. `SGLANG_ENABLE_FUSED_QKNORM_ROPE=0` also disables
|
||||
this path. This kernel does not enable model-level BCG support for FLUX.2.
|
||||
|
||||
### Activation
|
||||
|
||||
| Operator | Backend | Contract | Replaces |
|
||||
|
||||
@@ -573,6 +573,8 @@ _EXPORTS: dict[str, str] = {
|
||||
"try_fused_bias_mul_add": "sglang.kernels.kda_kernels.norm_scale_shift_jit",
|
||||
"try_fused_bias_scale_residual_norm_scale_shift": "sglang.kernels.kda_kernels.norm_scale_shift_jit",
|
||||
"triton_one_pass_rms_norm": "norm.rmsnorm_onepass_triton",
|
||||
"can_use_flux2_strided_qknorm_rope": "rope.flux2_qknorm_rope_triton",
|
||||
"flux2_strided_qknorm_rope": "rope.flux2_qknorm_rope_triton",
|
||||
"can_use_rmsnorm_preserve_reduction": "norm.rmsnorm_preserve_reduction",
|
||||
"rmsnorm_preserve_reduction": "norm.rmsnorm_preserve_reduction",
|
||||
"can_use_fused_rmsnorm_scale_shift": "norm.rmsnorm_scale_shift_bitexact",
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
"""Packed FLUX.2 Q/K RMSNorm and interleaved RoPE with native rounding."""
|
||||
|
||||
import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
|
||||
def can_use_flux2_strided_qknorm_rope(q, k, q_weight, k_weight, cache) -> bool:
|
||||
if (
|
||||
torch.compiler.is_compiling()
|
||||
or torch.is_grad_enabled()
|
||||
or torch.version.hip is not None
|
||||
or not q.is_cuda
|
||||
or torch.cuda.get_device_capability(q.device) != (9, 0)
|
||||
or q.ndim != 4
|
||||
or q.shape[-2:] != (24, 128)
|
||||
or q.numel() == 0
|
||||
or q.is_contiguous()
|
||||
or k.shape != q.shape
|
||||
or k.stride() != q.stride()
|
||||
or q.stride(-1) != 1
|
||||
or q.stride(-2) != 128
|
||||
or q.stride(0) != q.shape[1] * q.stride(1)
|
||||
or q.stride(1) < 24 * 128
|
||||
):
|
||||
return False
|
||||
if any(
|
||||
t.device != q.device or t.dtype != torch.bfloat16
|
||||
for t in (q, k, q_weight, k_weight)
|
||||
):
|
||||
return False
|
||||
if any(t.shape != (128,) or not t.is_contiguous() for t in (q_weight, k_weight)):
|
||||
return False
|
||||
return (
|
||||
isinstance(cache, torch.Tensor)
|
||||
and cache.device == q.device
|
||||
and cache.dtype == torch.float32
|
||||
and cache.ndim == 2
|
||||
and cache.shape[0] >= q.shape[1]
|
||||
and cache.shape[1] == 128
|
||||
and cache.is_contiguous()
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _flux2_strided_qknorm_rope_kernel(
|
||||
q_ptr,
|
||||
k_ptr,
|
||||
qw_ptr,
|
||||
kw_ptr,
|
||||
cache_ptr,
|
||||
qo_ptr,
|
||||
ko_ptr,
|
||||
ROWS: tl.constexpr,
|
||||
TOKENS: tl.constexpr,
|
||||
HEADS: tl.constexpr,
|
||||
TOKEN_STRIDE: tl.constexpr,
|
||||
CACHE_STRIDE: tl.constexpr,
|
||||
EPS: tl.constexpr,
|
||||
BLOCK_ROWS: tl.constexpr,
|
||||
):
|
||||
rows = tl.program_id(0) * BLOCK_ROWS + tl.arange(0, BLOCK_ROWS)[:, None]
|
||||
dims = tl.arange(0, 128)[None, :]
|
||||
if tl.program_id(1) == 0:
|
||||
x_ptr, w_ptr, y_ptr = q_ptr, qw_ptr, qo_ptr
|
||||
else:
|
||||
x_ptr, w_ptr, y_ptr = k_ptr, kw_ptr, ko_ptr
|
||||
offset = (rows // HEADS).to(tl.int64) * TOKEN_STRIDE + rows % HEADS * 128
|
||||
x = tl.load(x_ptr + offset + dims, mask=rows < ROWS, other=0.0).to(tl.float32)
|
||||
# Keep the exact arithmetic and row tiling of _rms_norm_tiled_onepass.
|
||||
mean_square = tl.sum(x * x, axis=1, keep_dims=True) / 128
|
||||
rstd = tl.math.rsqrt(mean_square + EPS)
|
||||
w = tl.load(w_ptr + dims)
|
||||
normalized = (x * rstd * w).to(y_ptr.dtype.element_ty).to(tl.float32)
|
||||
even, odd = tl.split(tl.reshape(normalized, (BLOCK_ROWS, 64, 2)))
|
||||
pos = rows // HEADS % TOKENS
|
||||
half = tl.arange(0, 64)[None, :]
|
||||
cos = tl.load(cache_ptr + pos * CACHE_STRIDE + half, mask=rows < ROWS, other=0.0)
|
||||
sin = tl.load(
|
||||
cache_ptr + pos * CACHE_STRIDE + 64 + half, mask=rows < ROWS, other=0.0
|
||||
)
|
||||
# FlashInfer rounds the sine product before the cosine multiply-add.
|
||||
# A subtraction from +0 can canonicalize signed zeros. Flip the IEEE
|
||||
# sign bit explicitly, matching FlashInfer's even-lane negation.
|
||||
negative_odd = (odd.to(tl.int32, bitcast=True) ^ -2147483648).to(
|
||||
tl.float32, bitcast=True
|
||||
)
|
||||
rotated_even = tl.fma(even, cos, negative_odd * sin)
|
||||
rotated_odd = tl.fma(odd, cos, even * sin)
|
||||
rotated = tl.reshape(tl.join(rotated_even, rotated_odd), (BLOCK_ROWS, 128))
|
||||
tl.store(y_ptr + rows * 128 + dims, rotated, mask=rows < ROWS)
|
||||
|
||||
|
||||
def flux2_strided_qknorm_rope(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
q_weight: torch.Tensor,
|
||||
k_weight: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
eps: float,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
batch, tokens, heads, dim = q.shape
|
||||
assert dim == 128 and k.shape == q.shape
|
||||
assert q.stride() == k.stride()
|
||||
assert q.stride(-1) == 1 and q.stride(-2) == 128
|
||||
assert q.stride(0) == tokens * q.stride(1)
|
||||
assert q.dtype == k.dtype == q_weight.dtype == k_weight.dtype == torch.bfloat16
|
||||
assert cos_sin_cache.dtype == torch.float32 and cos_sin_cache.shape[1] == 128
|
||||
output_q = torch.empty(q.shape, device=q.device, dtype=q.dtype)
|
||||
output_k = torch.empty_like(output_q)
|
||||
rows = batch * tokens * heads
|
||||
block_rows = min(16, triton.next_power_of_2(max(1, rows // 512)))
|
||||
with torch.cuda.device(q.device):
|
||||
_flux2_strided_qknorm_rope_kernel[(triton.cdiv(rows, block_rows), 2)](
|
||||
q,
|
||||
k,
|
||||
q_weight,
|
||||
k_weight,
|
||||
cos_sin_cache,
|
||||
output_q,
|
||||
output_k,
|
||||
rows,
|
||||
tokens,
|
||||
heads,
|
||||
q.stride(1),
|
||||
cos_sin_cache.stride(0),
|
||||
eps,
|
||||
BLOCK_ROWS=block_rows,
|
||||
)
|
||||
return output_q, output_k
|
||||
@@ -12,6 +12,7 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import os
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
import torch
|
||||
@@ -25,9 +26,11 @@ from sglang.kernels.ops.diffusion import (
|
||||
BitExactFusionGate,
|
||||
can_defer_flux2_gated_residual,
|
||||
can_use_flux2_gated_resnorm,
|
||||
can_use_flux2_strided_qknorm_rope,
|
||||
can_use_fused_layernorm_modulate,
|
||||
flux2_gated_resnorm_raw,
|
||||
flux2_nvfp4_swiglu_quant_active,
|
||||
flux2_strided_qknorm_rope,
|
||||
fused_layernorm_modulate_fp8_quant_raw,
|
||||
fused_layernorm_modulate_raw,
|
||||
fused_packed_silu_mul_bitexact,
|
||||
@@ -103,6 +106,68 @@ _FLUX2_LN_FP8 = BitExactFusionGate(
|
||||
)
|
||||
_FLUX2_LN_FP8_SIGS = _FLUX2_LN_FP8.verified_sigs
|
||||
assert _FLUX2_LN_FP8_SIGS is not None
|
||||
_FLUX2_STRIDED_QK_ROPE = BitExactFusionGate(
|
||||
"FLUX.2 Klein strided QK RMSNorm+RoPE", per_signature=True
|
||||
)
|
||||
|
||||
|
||||
def _flux2_single_qk_rope(q, k, q_norm, k_norm, head_dim, cache, complex_freqs):
|
||||
def reference():
|
||||
return apply_qk_norm_with_optional_rope(
|
||||
q=q,
|
||||
k=k,
|
||||
q_norm=q_norm,
|
||||
k_norm=k_norm,
|
||||
head_dim=head_dim,
|
||||
cos_sin_cache=cache,
|
||||
freqs_complex=complex_freqs,
|
||||
is_neox=False,
|
||||
allow_inplace=True,
|
||||
allow_strided_qk=current_platform.is_rocm(),
|
||||
)
|
||||
|
||||
gate = _FLUX2_STRIDED_QK_ROPE
|
||||
if (
|
||||
gate.disabled
|
||||
or head_dim != 128
|
||||
or type(q_norm) is not RMSNorm
|
||||
or type(k_norm) is not RMSNorm
|
||||
or q_norm.variance_epsilon != k_norm.variance_epsilon
|
||||
or any(
|
||||
norm.variance_size_override is not None
|
||||
or getattr(norm._forward_method, "__func__", None)
|
||||
is not RMSNorm.forward_cuda
|
||||
for norm in (q_norm, k_norm)
|
||||
)
|
||||
or os.getenv("SGLANG_ENABLE_FUSED_QKNORM_ROPE", "1").lower()
|
||||
in {"0", "false", "off", "no"}
|
||||
or not can_use_flux2_strided_qknorm_rope(
|
||||
q, k, q_norm.weight, k_norm.weight, cache
|
||||
)
|
||||
):
|
||||
return reference()
|
||||
sig = (q.device, q.shape, q.stride(), cache.stride(), q_norm.variance_epsilon)
|
||||
verified = gate.is_verified(sig)
|
||||
if not verified and torch.cuda.is_current_stream_capturing():
|
||||
return reference()
|
||||
try:
|
||||
out = flux2_strided_qknorm_rope(
|
||||
q, k, q_norm.weight, k_norm.weight, cache, q_norm.variance_epsilon
|
||||
)
|
||||
except Exception as exc:
|
||||
gate.on_exception(exc, logger=logger)
|
||||
return reference()
|
||||
if verified:
|
||||
return out
|
||||
return gate.accept_or_fallback(
|
||||
out,
|
||||
reference(),
|
||||
sig=sig,
|
||||
equal=lambda a, b: all(
|
||||
torch.equal(x.view(torch.int16), y.view(torch.int16)) for x, y in zip(a, b)
|
||||
),
|
||||
logger=logger,
|
||||
)
|
||||
|
||||
|
||||
def _valid_modelopt_fp8_linear(linear: nn.Module) -> bool:
|
||||
@@ -948,19 +1013,16 @@ class Flux2ParallelSelfAttention(torch.nn.Module, AttentionModuleMixin):
|
||||
if complex_freqs is not None:
|
||||
complex_freqs = complex_freqs[: query.shape[1]]
|
||||
|
||||
# QK-norm (+ RoPE) via the shared helper so the fused kernel path is used
|
||||
# here too — the single-stream block previously ran norm and RoPE as separate ops.
|
||||
query, key = apply_qk_norm_with_optional_rope(
|
||||
q=query,
|
||||
k=key,
|
||||
q_norm=self.norm_q,
|
||||
k_norm=self.norm_k,
|
||||
head_dim=self.head_dim,
|
||||
cos_sin_cache=cos_sin_cache,
|
||||
freqs_complex=complex_freqs,
|
||||
is_neox=False,
|
||||
allow_inplace=True,
|
||||
allow_strided_qk=True if current_platform.is_rocm() else False,
|
||||
# Packed QKV/MLP projection views can be normalized and rotated
|
||||
# without materializing contiguous Q/K inputs on the supported path.
|
||||
query, key = _flux2_single_qk_rope(
|
||||
query,
|
||||
key,
|
||||
self.norm_q,
|
||||
self.norm_k,
|
||||
self.head_dim,
|
||||
cos_sin_cache,
|
||||
complex_freqs,
|
||||
)
|
||||
hidden_states = self.attn(
|
||||
query,
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
"""Klein packed Q/K fusion against the actual native RMSNorm/RoPE dispatch."""
|
||||
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.diffusion import (
|
||||
BitExactFusionGate,
|
||||
can_use_flux2_strided_qknorm_rope,
|
||||
flux2_strided_qknorm_rope,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.layernorm import (
|
||||
RMSNorm,
|
||||
apply_qk_norm_with_optional_rope,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.dits import flux_2
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cuda_ci(est_time=25, stage="base-b-kernel-unit", runner_config="1-gpu-large")
|
||||
|
||||
|
||||
def inputs(batch, tokens):
|
||||
packed = torch.randn(batch, tokens, 9 * 3072, device="cuda", dtype=torch.bfloat16)
|
||||
q, k, _ = packed[:, :, : 3 * 3072].chunk(3, dim=-1)
|
||||
q, k = (x.unflatten(-1, (24, 128)) for x in (q, k))
|
||||
qn, kn = (RMSNorm(128, eps=1e-6).to("cuda", torch.bfloat16) for _ in range(2))
|
||||
qn.weight.normal_()
|
||||
kn.weight.normal_()
|
||||
angles = torch.randn(tokens, 64, device="cuda")
|
||||
cache = torch.cat((angles.cos(), angles.sin()), dim=-1)
|
||||
return packed, q, k, qn, kn, cache
|
||||
|
||||
|
||||
def reference(q, k, qn, kn, cache):
|
||||
return apply_qk_norm_with_optional_rope(
|
||||
q,
|
||||
k,
|
||||
qn,
|
||||
kn,
|
||||
128,
|
||||
cache,
|
||||
is_neox=False,
|
||||
allow_inplace=True,
|
||||
allow_strided_qk=False,
|
||||
)
|
||||
|
||||
|
||||
@unittest.skipUnless(
|
||||
torch.cuda.is_available() and torch.version.hip is None, "NVIDIA CUDA required"
|
||||
)
|
||||
class TestFlux2StridedQKNormRoPE(CustomTestCase):
|
||||
def assert_bits(self, actual, expected):
|
||||
for a, b in zip(actual, expected, strict=True):
|
||||
self.assertEqual(a.dtype, b.dtype)
|
||||
self.assertTrue(torch.equal(a.view(torch.int16), b.view(torch.int16)))
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_native_shapes_magnitudes_and_untouched_packed_input(self):
|
||||
torch.manual_seed(42)
|
||||
for batch, tokens in [(1, 17), (2, 257), (1, 4608)]:
|
||||
packed, q, k, qn, kn, cache = inputs(batch, tokens)
|
||||
for magnitude in (0.005, 1.0, 200.0):
|
||||
with self.subTest(batch=batch, tokens=tokens, magnitude=magnitude):
|
||||
packed.normal_().mul_(magnitude)
|
||||
original = packed.clone()
|
||||
gate = BitExactFusionGate("test Klein QK", per_signature=True)
|
||||
with patch.object(flux_2, "_FLUX2_STRIDED_QK_ROPE", gate):
|
||||
actual = flux_2._flux2_single_qk_rope(
|
||||
q, k, qn, kn, 128, cache, None
|
||||
)
|
||||
self.assert_bits(actual, reference(q, k, qn, kn, cache))
|
||||
self.assertTrue(
|
||||
torch.equal(
|
||||
packed.view(torch.int16), original.view(torch.int16)
|
||||
)
|
||||
)
|
||||
if can_use_flux2_strided_qknorm_rope(
|
||||
q, k, qn.weight, kn.weight, cache
|
||||
):
|
||||
self.assertTrue(gate.verified)
|
||||
self.assertFalse(gate.disabled)
|
||||
self.assert_bits(
|
||||
flux2_strided_qknorm_rope(
|
||||
q, k, qn.weight, kn.weight, cache, qn.variance_epsilon
|
||||
),
|
||||
reference(q, k, qn, kn, cache),
|
||||
)
|
||||
else:
|
||||
self.assertFalse(gate.verified)
|
||||
self.assertFalse(gate.disabled)
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_changed_inputs_weights_cache_and_graph_replay(self):
|
||||
packed, q, k, qn, kn, cache = inputs(1, 513)
|
||||
gate = BitExactFusionGate("test Klein replay", per_signature=True)
|
||||
with patch.object(flux_2, "_FLUX2_STRIDED_QK_ROPE", gate):
|
||||
flux_2._flux2_single_qk_rope(q, k, qn, kn, 128, cache, None)
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
out = flux_2._flux2_single_qk_rope(q, k, qn, kn, 128, cache, None)
|
||||
for seed in range(8):
|
||||
torch.manual_seed(seed)
|
||||
packed.normal_()
|
||||
qn.weight.normal_()
|
||||
kn.weight.normal_()
|
||||
cache.normal_()
|
||||
graph.replay()
|
||||
self.assert_bits(out, reference(q, k, qn, kn, cache))
|
||||
packed.zero_().neg_()
|
||||
graph.replay()
|
||||
self.assert_bits(out, reference(q, k, qn, kn, cache))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user