[diffusion] Fuse lossless Klein packed QK RMSNorm and RoPE on Hopper (#40490)
This commit is contained in:
@@ -573,6 +573,8 @@ _EXPORTS: dict[str, str] = {
|
||||
"try_fused_bias_mul_add": "sglang.kernels.kda_kernels.norm_scale_shift_jit",
|
||||
"try_fused_bias_scale_residual_norm_scale_shift": "sglang.kernels.kda_kernels.norm_scale_shift_jit",
|
||||
"triton_one_pass_rms_norm": "norm.rmsnorm_onepass_triton",
|
||||
"can_use_flux2_strided_qknorm_rope": "rope.flux2_qknorm_rope_triton",
|
||||
"flux2_strided_qknorm_rope": "rope.flux2_qknorm_rope_triton",
|
||||
"can_use_rmsnorm_preserve_reduction": "norm.rmsnorm_preserve_reduction",
|
||||
"rmsnorm_preserve_reduction": "norm.rmsnorm_preserve_reduction",
|
||||
"can_use_fused_rmsnorm_scale_shift": "norm.rmsnorm_scale_shift_bitexact",
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
"""Packed FLUX.2 Q/K RMSNorm and interleaved RoPE with native rounding."""
|
||||
|
||||
import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
|
||||
def can_use_flux2_strided_qknorm_rope(q, k, q_weight, k_weight, cache) -> bool:
|
||||
if (
|
||||
torch.compiler.is_compiling()
|
||||
or torch.is_grad_enabled()
|
||||
or torch.version.hip is not None
|
||||
or not q.is_cuda
|
||||
or torch.cuda.get_device_capability(q.device) != (9, 0)
|
||||
or q.ndim != 4
|
||||
or q.shape[-2:] != (24, 128)
|
||||
or q.numel() == 0
|
||||
or q.is_contiguous()
|
||||
or k.shape != q.shape
|
||||
or k.stride() != q.stride()
|
||||
or q.stride(-1) != 1
|
||||
or q.stride(-2) != 128
|
||||
or q.stride(0) != q.shape[1] * q.stride(1)
|
||||
or q.stride(1) < 24 * 128
|
||||
):
|
||||
return False
|
||||
if any(
|
||||
t.device != q.device or t.dtype != torch.bfloat16
|
||||
for t in (q, k, q_weight, k_weight)
|
||||
):
|
||||
return False
|
||||
if any(t.shape != (128,) or not t.is_contiguous() for t in (q_weight, k_weight)):
|
||||
return False
|
||||
return (
|
||||
isinstance(cache, torch.Tensor)
|
||||
and cache.device == q.device
|
||||
and cache.dtype == torch.float32
|
||||
and cache.ndim == 2
|
||||
and cache.shape[0] >= q.shape[1]
|
||||
and cache.shape[1] == 128
|
||||
and cache.is_contiguous()
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _flux2_strided_qknorm_rope_kernel(
|
||||
q_ptr,
|
||||
k_ptr,
|
||||
qw_ptr,
|
||||
kw_ptr,
|
||||
cache_ptr,
|
||||
qo_ptr,
|
||||
ko_ptr,
|
||||
ROWS: tl.constexpr,
|
||||
TOKENS: tl.constexpr,
|
||||
HEADS: tl.constexpr,
|
||||
TOKEN_STRIDE: tl.constexpr,
|
||||
CACHE_STRIDE: tl.constexpr,
|
||||
EPS: tl.constexpr,
|
||||
BLOCK_ROWS: tl.constexpr,
|
||||
):
|
||||
rows = tl.program_id(0) * BLOCK_ROWS + tl.arange(0, BLOCK_ROWS)[:, None]
|
||||
dims = tl.arange(0, 128)[None, :]
|
||||
if tl.program_id(1) == 0:
|
||||
x_ptr, w_ptr, y_ptr = q_ptr, qw_ptr, qo_ptr
|
||||
else:
|
||||
x_ptr, w_ptr, y_ptr = k_ptr, kw_ptr, ko_ptr
|
||||
offset = (rows // HEADS).to(tl.int64) * TOKEN_STRIDE + rows % HEADS * 128
|
||||
x = tl.load(x_ptr + offset + dims, mask=rows < ROWS, other=0.0).to(tl.float32)
|
||||
# Keep the exact arithmetic and row tiling of _rms_norm_tiled_onepass.
|
||||
mean_square = tl.sum(x * x, axis=1, keep_dims=True) / 128
|
||||
rstd = tl.math.rsqrt(mean_square + EPS)
|
||||
w = tl.load(w_ptr + dims)
|
||||
normalized = (x * rstd * w).to(y_ptr.dtype.element_ty).to(tl.float32)
|
||||
even, odd = tl.split(tl.reshape(normalized, (BLOCK_ROWS, 64, 2)))
|
||||
pos = rows // HEADS % TOKENS
|
||||
half = tl.arange(0, 64)[None, :]
|
||||
cos = tl.load(cache_ptr + pos * CACHE_STRIDE + half, mask=rows < ROWS, other=0.0)
|
||||
sin = tl.load(
|
||||
cache_ptr + pos * CACHE_STRIDE + 64 + half, mask=rows < ROWS, other=0.0
|
||||
)
|
||||
# FlashInfer rounds the sine product before the cosine multiply-add.
|
||||
# A subtraction from +0 can canonicalize signed zeros. Flip the IEEE
|
||||
# sign bit explicitly, matching FlashInfer's even-lane negation.
|
||||
negative_odd = (odd.to(tl.int32, bitcast=True) ^ -2147483648).to(
|
||||
tl.float32, bitcast=True
|
||||
)
|
||||
rotated_even = tl.fma(even, cos, negative_odd * sin)
|
||||
rotated_odd = tl.fma(odd, cos, even * sin)
|
||||
rotated = tl.reshape(tl.join(rotated_even, rotated_odd), (BLOCK_ROWS, 128))
|
||||
tl.store(y_ptr + rows * 128 + dims, rotated, mask=rows < ROWS)
|
||||
|
||||
|
||||
def flux2_strided_qknorm_rope(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
q_weight: torch.Tensor,
|
||||
k_weight: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
eps: float,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
batch, tokens, heads, dim = q.shape
|
||||
assert dim == 128 and k.shape == q.shape
|
||||
assert q.stride() == k.stride()
|
||||
assert q.stride(-1) == 1 and q.stride(-2) == 128
|
||||
assert q.stride(0) == tokens * q.stride(1)
|
||||
assert q.dtype == k.dtype == q_weight.dtype == k_weight.dtype == torch.bfloat16
|
||||
assert cos_sin_cache.dtype == torch.float32 and cos_sin_cache.shape[1] == 128
|
||||
output_q = torch.empty(q.shape, device=q.device, dtype=q.dtype)
|
||||
output_k = torch.empty_like(output_q)
|
||||
rows = batch * tokens * heads
|
||||
block_rows = min(16, triton.next_power_of_2(max(1, rows // 512)))
|
||||
with torch.cuda.device(q.device):
|
||||
_flux2_strided_qknorm_rope_kernel[(triton.cdiv(rows, block_rows), 2)](
|
||||
q,
|
||||
k,
|
||||
q_weight,
|
||||
k_weight,
|
||||
cos_sin_cache,
|
||||
output_q,
|
||||
output_k,
|
||||
rows,
|
||||
tokens,
|
||||
heads,
|
||||
q.stride(1),
|
||||
cos_sin_cache.stride(0),
|
||||
eps,
|
||||
BLOCK_ROWS=block_rows,
|
||||
)
|
||||
return output_q, output_k
|
||||
@@ -12,6 +12,7 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import os
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
import torch
|
||||
@@ -25,9 +26,11 @@ from sglang.kernels.ops.diffusion import (
|
||||
BitExactFusionGate,
|
||||
can_defer_flux2_gated_residual,
|
||||
can_use_flux2_gated_resnorm,
|
||||
can_use_flux2_strided_qknorm_rope,
|
||||
can_use_fused_layernorm_modulate,
|
||||
flux2_gated_resnorm_raw,
|
||||
flux2_nvfp4_swiglu_quant_active,
|
||||
flux2_strided_qknorm_rope,
|
||||
fused_layernorm_modulate_fp8_quant_raw,
|
||||
fused_layernorm_modulate_raw,
|
||||
fused_packed_silu_mul_bitexact,
|
||||
@@ -103,6 +106,68 @@ _FLUX2_LN_FP8 = BitExactFusionGate(
|
||||
)
|
||||
_FLUX2_LN_FP8_SIGS = _FLUX2_LN_FP8.verified_sigs
|
||||
assert _FLUX2_LN_FP8_SIGS is not None
|
||||
_FLUX2_STRIDED_QK_ROPE = BitExactFusionGate(
|
||||
"FLUX.2 Klein strided QK RMSNorm+RoPE", per_signature=True
|
||||
)
|
||||
|
||||
|
||||
def _flux2_single_qk_rope(q, k, q_norm, k_norm, head_dim, cache, complex_freqs):
|
||||
def reference():
|
||||
return apply_qk_norm_with_optional_rope(
|
||||
q=q,
|
||||
k=k,
|
||||
q_norm=q_norm,
|
||||
k_norm=k_norm,
|
||||
head_dim=head_dim,
|
||||
cos_sin_cache=cache,
|
||||
freqs_complex=complex_freqs,
|
||||
is_neox=False,
|
||||
allow_inplace=True,
|
||||
allow_strided_qk=current_platform.is_rocm(),
|
||||
)
|
||||
|
||||
gate = _FLUX2_STRIDED_QK_ROPE
|
||||
if (
|
||||
gate.disabled
|
||||
or head_dim != 128
|
||||
or type(q_norm) is not RMSNorm
|
||||
or type(k_norm) is not RMSNorm
|
||||
or q_norm.variance_epsilon != k_norm.variance_epsilon
|
||||
or any(
|
||||
norm.variance_size_override is not None
|
||||
or getattr(norm._forward_method, "__func__", None)
|
||||
is not RMSNorm.forward_cuda
|
||||
for norm in (q_norm, k_norm)
|
||||
)
|
||||
or os.getenv("SGLANG_ENABLE_FUSED_QKNORM_ROPE", "1").lower()
|
||||
in {"0", "false", "off", "no"}
|
||||
or not can_use_flux2_strided_qknorm_rope(
|
||||
q, k, q_norm.weight, k_norm.weight, cache
|
||||
)
|
||||
):
|
||||
return reference()
|
||||
sig = (q.device, q.shape, q.stride(), cache.stride(), q_norm.variance_epsilon)
|
||||
verified = gate.is_verified(sig)
|
||||
if not verified and torch.cuda.is_current_stream_capturing():
|
||||
return reference()
|
||||
try:
|
||||
out = flux2_strided_qknorm_rope(
|
||||
q, k, q_norm.weight, k_norm.weight, cache, q_norm.variance_epsilon
|
||||
)
|
||||
except Exception as exc:
|
||||
gate.on_exception(exc, logger=logger)
|
||||
return reference()
|
||||
if verified:
|
||||
return out
|
||||
return gate.accept_or_fallback(
|
||||
out,
|
||||
reference(),
|
||||
sig=sig,
|
||||
equal=lambda a, b: all(
|
||||
torch.equal(x.view(torch.int16), y.view(torch.int16)) for x, y in zip(a, b)
|
||||
),
|
||||
logger=logger,
|
||||
)
|
||||
|
||||
|
||||
def _valid_modelopt_fp8_linear(linear: nn.Module) -> bool:
|
||||
@@ -948,19 +1013,16 @@ class Flux2ParallelSelfAttention(torch.nn.Module, AttentionModuleMixin):
|
||||
if complex_freqs is not None:
|
||||
complex_freqs = complex_freqs[: query.shape[1]]
|
||||
|
||||
# QK-norm (+ RoPE) via the shared helper so the fused kernel path is used
|
||||
# here too — the single-stream block previously ran norm and RoPE as separate ops.
|
||||
query, key = apply_qk_norm_with_optional_rope(
|
||||
q=query,
|
||||
k=key,
|
||||
q_norm=self.norm_q,
|
||||
k_norm=self.norm_k,
|
||||
head_dim=self.head_dim,
|
||||
cos_sin_cache=cos_sin_cache,
|
||||
freqs_complex=complex_freqs,
|
||||
is_neox=False,
|
||||
allow_inplace=True,
|
||||
allow_strided_qk=True if current_platform.is_rocm() else False,
|
||||
# Packed QKV/MLP projection views can be normalized and rotated
|
||||
# without materializing contiguous Q/K inputs on the supported path.
|
||||
query, key = _flux2_single_qk_rope(
|
||||
query,
|
||||
key,
|
||||
self.norm_q,
|
||||
self.norm_k,
|
||||
self.head_dim,
|
||||
cos_sin_cache,
|
||||
complex_freqs,
|
||||
)
|
||||
hidden_states = self.attn(
|
||||
query,
|
||||
|
||||
Reference in New Issue
Block a user