[diffusion] Qwen-Image 2.1: fuse Q/K RMSNorm + RoPE + KV packing into one CUDA kernel and project Q/K/V with one packed GEMM (#41339)

This commit is contained in:
triple-mu
2026-09-28 21:45:39 +08:00
committed by GitHub
parent 8151378c1a
commit 2e7e0802f4
9 changed files with 1497 additions and 12 deletions
@@ -0,0 +1,344 @@
// Bit-exact 128-wide RMSNorm + complex RoPE for Qwen-Image-2.1 attention prep.
//
// Eager chain reproduced for bf16 activations:
// RMSNorm(cast_x_before_out_mul=True): var = mean(float(x)^2) in aten's
// vectorized 128-wide fp32 mean order (lane t owns elements 4t..4t+3, sums
// them left to right, then a 32-lane shuffle tree with offsets 16,8,4,2,1),
// inv = rsqrtf(var + eps), n = bf16(x * inv), o = bf16(n * w);
// complex RoPE on interleaved pairs: out = view_as_real(complex(o) * rope),
// with the FMA orientation PyTorch's complex64 multiply uses on this GPU
// (kFuseRealSin, probed on the Python side).
// One launch handles the Q rows and the K rows (source -> destination, in place
// when they alias), copies the cached prefix K/V into rows [0:P] of the
// [B, P+S, H, 128] K/V buffers and, with kCopyV, the raw V rows into [P:P+S].
#pragma once
#include <sgl_kernel/tensor.h>
#include <sgl_kernel/utils.h>
#include <sgl_kernel/type.cuh>
#include <sgl_kernel/utils.cuh>
#include <sgl_kernel/vec.cuh>
#include <algorithm>
#include <cstdint>
namespace sglang {
namespace qknorm_complex_rope {
constexpr int kHeadDim = 128;
constexpr int kLaneElems = kHeadDim / device::kWarpThreads; // 4 bf16 per lane
constexpr uint32_t kThreads = 256;
constexpr uint32_t kWarpsPerBlock = kThreads / device::kWarpThreads;
constexpr uint32_t kMaxBlocks = 65535;
constexpr int64_t kVecAlignBytes = 8;
static_assert(kLaneElems == 4, "the aten mean replica assumes four elements per lane");
using RowVec = device::AlignedVector<bf16_t, kLaneElems>; // 8 bytes
using RopeVec = device::AlignedVector<fp32_t, kLaneElems>; // cos0 sin0 cos1 sin1
/// Strided [B, T, H, 128] tensor: element (b, t, h, d) lives at ptr + b*batch_stride + t*token_stride + h*128 + d.
struct Rows {
const bf16_t* ptr;
int64_t batch_stride;
int64_t token_stride;
};
struct MutableRows {
bf16_t* ptr;
int64_t batch_stride;
int64_t token_stride;
};
struct Params {
Rows q_src;
MutableRows q_dst;
Rows k_src;
MutableRows k_dst; // [B, P+S, H, 128]; token rows land at P + s
MutableRows v_dst; // [B, P+S, H, 128]
Rows v_src; // read only with kCopyV
const bf16_t* q_weight;
const bf16_t* k_weight;
const bf16_t* k_prefix; // contiguous [B, P, H, 128]
const bf16_t* v_prefix; // contiguous [B, P, H, 128]
const fp32_t* rope; // [S, 128] = view_as_real of the complex64 [S, 64] table
uint32_t batch;
uint32_t seq;
uint32_t heads;
uint32_t prefix;
uint32_t k_rows_enabled; // 0 for the single-tensor entry point
float eps;
};
SGL_DEVICE const bf16_t* row_ptr(const Rows& rows, int64_t b, int64_t t, int64_t h) {
return rows.ptr + b * rows.batch_stride + t * rows.token_stride + h * kHeadDim;
}
SGL_DEVICE bf16_t* row_ptr(const MutableRows& rows, int64_t b, int64_t t, int64_t h) {
return rows.ptr + b * rows.batch_stride + t * rows.token_stride + h * kHeadDim;
}
/// \brief Normalize and rotate one 128-wide row; every lane owns 4 consecutive elements.
template <bool kFuseRealSin>
SGL_DEVICE void norm_rope_row(
const bf16_t* __restrict__ src,
bf16_t* __restrict__ dst,
const bf16_t* __restrict__ weight,
const fp32_t* __restrict__ rope_row,
float eps,
int lane) {
RowVec x;
RowVec w;
RopeVec r;
x.load(src + lane * kLaneElems);
w.load(weight + lane * kLaneElems);
r.load(rope_row + lane * kLaneElems);
float value[kLaneElems];
#pragma unroll
for (int i = 0; i < kLaneElems; ++i) {
value[i] = device::cast<fp32_t>(x[i]);
}
// aten vectorized mean: four squares combined left to right ...
float part = __fmul_rn(value[0], value[0]);
part = __fadd_rn(part, __fmul_rn(value[1], value[1]));
part = __fadd_rn(part, __fmul_rn(value[2], value[2]));
part = __fadd_rn(part, __fmul_rn(value[3], value[3]));
// ... then the 32-lane tree with decreasing offsets (matches shfl_down 16..1 for lane 0).
#pragma unroll
for (int offset = device::kWarpThreads / 2; offset > 0; offset >>= 1) {
part = __fadd_rn(part, __shfl_xor_sync(0xffffffffu, part, offset));
}
const float variance = __fmul_rn(part, 1.0f / kHeadDim);
const float inv = rsqrtf(__fadd_rn(variance, eps));
// cast_x_before_out_mul: round after the normalize, then again after the weight.
float out_f32[kLaneElems];
#pragma unroll
for (int i = 0; i < kLaneElems; ++i) {
const float normalized = device::cast<fp32_t>(device::cast<bf16_t>(__fmul_rn(value[i], inv)));
out_f32[i] = device::cast<fp32_t>(device::cast<bf16_t>(__fmul_rn(normalized, device::cast<fp32_t>(w[i]))));
}
RowVec out;
#pragma unroll
for (int p = 0; p < kLaneElems / 2; ++p) {
const float re = out_f32[2 * p];
const float im = out_f32[2 * p + 1];
const float cos = r[2 * p];
const float sin = r[2 * p + 1];
const float out_re = __fmaf_rn(re, cos, -__fmul_rn(im, sin));
float out_im;
if constexpr (kFuseRealSin) {
out_im = __fmaf_rn(re, sin, __fmul_rn(im, cos));
} else {
out_im = __fmaf_rn(im, cos, __fmul_rn(re, sin));
}
out[2 * p] = device::cast<bf16_t>(out_re);
out[2 * p + 1] = device::cast<bf16_t>(out_im);
}
out.store(dst + lane * kLaneElems);
}
SGL_DEVICE void copy_row(const bf16_t* __restrict__ src, bf16_t* __restrict__ dst, int lane) {
RowVec v;
v.load(src + lane * kLaneElems);
v.store(dst + lane * kLaneElems);
}
/// Work list: Q rows, K rows, prefix-K copies, prefix-V copies, then (kCopyV) V copies. One warp per row.
template <bool kFuseRealSin, bool kCopyV>
__global__ void kernel(const Params __grid_constant__ params) {
const int lane = threadIdx.x & (device::kWarpThreads - 1);
const uint32_t warp = threadIdx.x >> 5;
const int64_t heads = params.heads;
const int64_t seq = params.seq;
const int64_t prefix = params.prefix;
const int64_t rows_q = static_cast<int64_t>(params.batch) * seq * heads;
const int64_t rows_k = params.k_rows_enabled ? rows_q : 0;
const int64_t rows_p = params.k_rows_enabled ? static_cast<int64_t>(params.batch) * prefix * heads : 0;
const int64_t rows_v = kCopyV ? rows_q : 0;
const int64_t total = rows_q + rows_k + 2 * rows_p + rows_v;
const int64_t stride = static_cast<int64_t>(gridDim.x) * kWarpsPerBlock;
for (int64_t row = static_cast<int64_t>(blockIdx.x) * kWarpsPerBlock + warp; row < total; row += stride) {
int64_t r = row;
if (r < rows_q) {
const int64_t h = r % heads, t = r / heads, s = t % seq, b = t / seq;
norm_rope_row<kFuseRealSin>(
row_ptr(params.q_src, b, s, h),
row_ptr(params.q_dst, b, s, h),
params.q_weight,
params.rope + s * kHeadDim,
params.eps,
lane);
continue;
}
r -= rows_q;
if (r < rows_k) {
const int64_t h = r % heads, t = r / heads, s = t % seq, b = t / seq;
norm_rope_row<kFuseRealSin>(
row_ptr(params.k_src, b, s, h),
row_ptr(params.k_dst, b, prefix + s, h),
params.k_weight,
params.rope + s * kHeadDim,
params.eps,
lane);
continue;
}
r -= rows_k;
if (r < rows_p) {
const int64_t h = r % heads, t = r / heads, p = t % prefix, b = t / prefix;
copy_row(params.k_prefix + ((b * prefix + p) * heads + h) * kHeadDim, row_ptr(params.k_dst, b, p, h), lane);
continue;
}
r -= rows_p;
if (r < rows_p) {
const int64_t h = r % heads, t = r / heads, p = t % prefix, b = t / prefix;
copy_row(params.v_prefix + ((b * prefix + p) * heads + h) * kHeadDim, row_ptr(params.v_dst, b, p, h), lane);
continue;
}
if constexpr (kCopyV) {
r -= rows_p;
const int64_t h = r % heads, t = r / heads, s = t % seq, b = t / seq;
copy_row(row_ptr(params.v_src, b, s, h), row_ptr(params.v_dst, b, prefix + s, h), lane);
}
}
}
template <bool kFuseRealSin, bool kCopyV>
struct Kernel {
/// \brief Single tensor, out of place: `out = rope(rmsnorm(x))` for a contiguous [B, S, H, 128] bf16 tensor.
static void
run(tvm::ffi::TensorView x,
tvm::ffi::TensorView out,
tvm::ffi::TensorView weight,
tvm::ffi::TensorView rope,
double eps) {
using namespace host;
auto B = SymbolicSize{"batch"};
auto S = SymbolicSize{"seq"};
auto H = SymbolicSize{"heads"};
auto device = SymbolicDevice{};
device.set_options<kDLCUDA>();
TensorMatcher({B, S, H, kHeadDim})
.with_dtype<bf16_t>()
.with_device(device)
.ensure_alignment(kVecAlignBytes)
.verify(x)
.verify(out);
TensorMatcher({kHeadDim}).with_dtype<bf16_t>().with_device(device).verify(weight);
TensorMatcher({S, kHeadDim}).with_dtype<fp32_t>().with_device(device).ensure_alignment(16).verify(rope);
const int64_t batch = B.unwrap(), seq = S.unwrap(), heads = H.unwrap();
if (batch == 0 || seq == 0 || heads == 0) return;
const int64_t token_stride = heads * kHeadDim;
Params params{};
params.q_src = Rows{static_cast<const bf16_t*>(x.data_ptr()), seq * token_stride, token_stride};
params.q_dst = MutableRows{static_cast<bf16_t*>(out.data_ptr()), seq * token_stride, token_stride};
params.q_weight = static_cast<const bf16_t*>(weight.data_ptr());
params.k_weight = params.q_weight;
params.rope = static_cast<const fp32_t*>(rope.data_ptr());
params.batch = static_cast<uint32_t>(batch);
params.seq = static_cast<uint32_t>(seq);
params.heads = static_cast<uint32_t>(heads);
params.prefix = 0;
params.k_rows_enabled = 0;
params.eps = static_cast<float>(eps);
launch(params, batch * seq * heads, device.unwrap());
}
/// \brief Q in place, K source -> rows [P:] of k_out, prefix K/V -> rows [0:P] of k_out/v_out,
/// raw V -> rows [P:] of v_out when kCopyV (v_src is ignored otherwise).
static void run_pack(
tvm::ffi::TensorView q,
tvm::ffi::TensorView q_weight,
tvm::ffi::TensorView k_src,
tvm::ffi::TensorView k_out,
tvm::ffi::TensorView k_weight,
tvm::ffi::TensorView v_out,
tvm::ffi::TensorView v_src,
tvm::ffi::TensorView rope,
tvm::ffi::TensorView k_prefix,
tvm::ffi::TensorView v_prefix,
double eps) {
using namespace host;
auto B = SymbolicSize{"batch"};
auto S = SymbolicSize{"seq"};
auto H = SymbolicSize{"heads"};
auto P = SymbolicSize{"prefix"};
auto PS = SymbolicSize{"prefix_plus_seq"};
auto Bq = SymbolicSize{"q_batch_stride"}, Tq = SymbolicSize{"q_token_stride"};
auto Bk = SymbolicSize{"k_src_batch_stride"}, Tk = SymbolicSize{"k_src_token_stride"};
auto Bko = SymbolicSize{"k_out_batch_stride"}, Tko = SymbolicSize{"k_out_token_stride"};
auto Bvo = SymbolicSize{"v_out_batch_stride"}, Tvo = SymbolicSize{"v_out_token_stride"};
auto Bv = SymbolicSize{"v_src_batch_stride"}, Tv = SymbolicSize{"v_src_token_stride"};
auto device = SymbolicDevice{};
device.set_options<kDLCUDA>();
// TensorMatcher is not copyable, so verify inside the lambda; the member
// template call on a dependent expression needs `template`.
const auto verify_rows = [&](const tvm::ffi::TensorView& t, auto& rows, auto& batch_stride, auto& token_stride) {
TensorMatcher({B, rows, H, kHeadDim})
.with_strides({batch_stride, token_stride, kHeadDim, 1})
.template with_dtype<bf16_t>()
.with_device(device)
.ensure_alignment(kVecAlignBytes)
.verify(t);
};
verify_rows(q, S, Bq, Tq);
verify_rows(k_src, S, Bk, Tk);
verify_rows(k_out, PS, Bko, Tko);
verify_rows(v_out, PS, Bvo, Tvo);
if constexpr (kCopyV) {
verify_rows(v_src, S, Bv, Tv);
}
TensorMatcher({B, P, H, kHeadDim})
.with_dtype<bf16_t>()
.with_device(device)
.ensure_alignment(kVecAlignBytes)
.verify(k_prefix)
.verify(v_prefix);
TensorMatcher({kHeadDim}).with_dtype<bf16_t>().with_device(device).verify(q_weight).verify(k_weight);
TensorMatcher({S, kHeadDim}).with_dtype<fp32_t>().with_device(device).ensure_alignment(16).verify(rope);
const int64_t batch = B.unwrap(), seq = S.unwrap(), heads = H.unwrap(), prefix = P.unwrap();
CHECK_HOST(PS.unwrap() == prefix + seq) << "k_out/v_out must hold prefix + seq tokens";
if (batch == 0 || seq == 0 || heads == 0) return;
Params params{};
params.q_src = Rows{static_cast<const bf16_t*>(q.data_ptr()), Bq.unwrap(), Tq.unwrap()};
params.q_dst = MutableRows{static_cast<bf16_t*>(q.data_ptr()), Bq.unwrap(), Tq.unwrap()};
params.k_src = Rows{static_cast<const bf16_t*>(k_src.data_ptr()), Bk.unwrap(), Tk.unwrap()};
params.k_dst = MutableRows{static_cast<bf16_t*>(k_out.data_ptr()), Bko.unwrap(), Tko.unwrap()};
params.v_dst = MutableRows{static_cast<bf16_t*>(v_out.data_ptr()), Bvo.unwrap(), Tvo.unwrap()};
if constexpr (kCopyV) {
params.v_src = Rows{static_cast<const bf16_t*>(v_src.data_ptr()), Bv.unwrap(), Tv.unwrap()};
}
params.q_weight = static_cast<const bf16_t*>(q_weight.data_ptr());
params.k_weight = static_cast<const bf16_t*>(k_weight.data_ptr());
params.k_prefix = static_cast<const bf16_t*>(k_prefix.data_ptr());
params.v_prefix = static_cast<const bf16_t*>(v_prefix.data_ptr());
params.rope = static_cast<const fp32_t*>(rope.data_ptr());
params.batch = static_cast<uint32_t>(batch);
params.seq = static_cast<uint32_t>(seq);
params.heads = static_cast<uint32_t>(heads);
params.prefix = static_cast<uint32_t>(prefix);
params.k_rows_enabled = 1;
params.eps = static_cast<float>(eps);
const int64_t rows = batch * heads * (2 * seq + 2 * prefix + (kCopyV ? seq : 0));
launch(params, rows, device.unwrap());
}
private:
static void launch(const Params& params, int64_t rows, DLDevice device) {
using namespace host;
const int64_t needed = div_ceil(rows, static_cast<int64_t>(kWarpsPerBlock));
const auto blocks = static_cast<uint32_t>(std::min<int64_t>(needed, kMaxBlocks));
LaunchKernel(blocks, kThreads, device)(kernel<kFuseRealSin, kCopyV>, params);
}
};
} // namespace qknorm_complex_rope
} // namespace sglang
@@ -147,6 +147,7 @@ tensor copy per residual site.
| `fused_rope_rotate_half_bitexact` | Triton | bit-exact (elementwise only) |
| `fused_complex_rope` | Triton | preserves CUDA complex64 multiply rounding for contiguous BSHD inputs; Qwen-Image 2.1 verifies its first call against eager |
| `rmsnorm_preserve_reduction` | Triton + aten | preserves the FP32 mean reduction and cast-before-weight rounding; fuses only pointwise work for contiguous FP16/BF16 inputs; Qwen-Image 2.1 verifies its first call |
| `qknorm_complex_rope_cuda` / `qknorm_complex_rope_pack_` | JIT CUDA | bit-exact vs `RMSNorm(cast_x_before_out_mul=True)` + complex64 RoPE for head_dim 128; the pack variant normalizes Q and K in place behind a prefix, copies prefix K/V (and optionally V) in the same launch, and accepts token-strided views of a packed QKV buffer; Qwen-Image 2.1 verifies its first call |
| `fused_interleaved_rope_fp64` | JIT CUDA | bit-exact vs paired SANA-Video fp64 RoPE |
| `fused_inplace_helios_qk_rope` | JIT CUDA | bit-exact paired in-place RoPE for Helios' transposed frequency layout |
| `ltx2_qknorm_split_rope_cuda` | KDA (JIT CUDA) | close; **validated on B200** |
@@ -371,6 +371,20 @@ _SPECS: tuple[tuple[str, KernelBackend, str, frozenset, str], ...] = (
_CUDA,
"Out-of-place fused QK-norm + RoPE (raw q/k preserved).",
),
(
"diffusion.qknorm_complex_rope_cuda",
KernelBackend.JIT,
"rope.qknorm_complex_rope_jit:qknorm_complex_rope_cuda",
_CUDA,
"Bit-exact 128-wide RMSNorm + complex RoPE (Qwen-Image 2.1), CUDA.",
),
(
"diffusion.qknorm_complex_rope_pack_",
KernelBackend.JIT,
"rope.qknorm_complex_rope_jit:qknorm_complex_rope_pack_",
_CUDA,
"Q/K RMSNorm + complex RoPE with prefix K/V packing into [B, P+S, H, 128] buffers, CUDA.",
),
(
"diffusion.vdn_delta_factors",
KernelBackend.JIT,
@@ -638,6 +652,10 @@ _EXPORTS: dict[str, str] = {
"apply_rotary_embedding": "rope.rotary_triton",
"can_use_fused_complex_rope": "rope.complex_rope_triton",
"fused_complex_rope": "rope.complex_rope_triton",
"can_use_qknorm_complex_rope_cuda": "rope.qknorm_complex_rope_jit",
"can_use_qknorm_complex_rope_pack": "rope.qknorm_complex_rope_jit",
"qknorm_complex_rope_cuda": "rope.qknorm_complex_rope_jit",
"qknorm_complex_rope_pack_": "rope.qknorm_complex_rope_jit",
# Tensor layout transformations fused with downstream quantization
"try_flux2_token_cat_fp8": "sglang.kernels.kda_kernels.flux2_token_cat_fp8_triton",
# Activation-function fusions
@@ -0,0 +1,230 @@
# SPDX-License-Identifier: Apache-2.0
"""Bit-exact CUDA RMSNorm + complex RoPE for Qwen-Image-2.1 attention prep.
One launch replaces the Triton ``qknorm_complex_rope`` (Q) and
``qknorm_complex_rope_kv`` (K, V, prefix) pair: it normalizes and rotates the Q
rows in place, writes the normalized/rotated K rows into rows ``[P:P+S]`` of a
``[B, P+S, H, 128]`` buffer (in place when the projection already wrote them
there), copies the cached prefix K/V into rows ``[0:P]`` and, only when asked,
the raw V rows into ``[P:P+S]``.
Numerical contract (bf16 in and out): fp32 squares summed in aten's vectorized
128-wide mean order (lane ``t`` owns elements ``4t..4t+3`` left to right, then
a 32-lane shuffle tree), ``rsqrtf(var + eps)``, bf16 rounding after the
normalize and after the weight multiply (``RMSNorm(cast_x_before_out_mul=True)``),
then explicit-FMA rotation in the orientation ``_fuse_real_sin`` probes for the
running GPU. Verified ``torch.equal`` against the eager chain on the production
shape ``[1, 4096, 32, 128]`` with an 18-token prefix and on ``[1, 18, 32, 128]``.
CUDA only: the ROCm reduction tree differs. Callers verify once through a
``BitExactFusionGate`` and keep the eager chain as the fallback.
"""
from __future__ import annotations
from typing import TYPE_CHECKING, Optional
import torch
from sglang.kernels.jit.utils import cache_once, load_jit, make_cpp_args
from sglang.kernels.ops.diffusion.rope.complex_rope_triton import _fuse_real_sin
from sglang.srt.utils.custom_op import register_custom_op
if TYPE_CHECKING:
from tvm_ffi.module import Module
_HEAD_DIM = 128
_ROW_ALIGN = 8
@cache_once
def _jit_module(fuse_real_sin: bool, copy_v: bool) -> Module:
args = make_cpp_args(fuse_real_sin, copy_v)
return load_jit(
"qknorm_complex_rope",
*args,
cuda_files=["diffusion/qknorm_complex_rope.cuh"],
cuda_wrappers=[
("norm_rope", f"qknorm_complex_rope::Kernel<{args}>::run"),
("norm_rope_pack", f"qknorm_complex_rope::Kernel<{args}>::run_pack"),
],
)
def _is_rows(t: torch.Tensor, device: torch.device, tokens: int | None = None) -> bool:
"""``[B, T, H, 128]`` bf16 CUDA tensor whose heads are contiguous; batch/token strides may be padded."""
return (
isinstance(t, torch.Tensor)
and t.dtype is torch.bfloat16
and t.is_cuda
and t.device == device
and t.dim() == 4
and t.numel() > 0
and t.shape[-1] == _HEAD_DIM
and (tokens is None or t.shape[1] == tokens)
and t.stride(-1) == 1
and t.stride(-2) == _HEAD_DIM
and t.stride(1) % (_ROW_ALIGN // 2) == 0
and t.stride(0) % (_ROW_ALIGN // 2) == 0
and t.data_ptr() % _ROW_ALIGN == 0
)
def _is_weight(w: torch.Tensor, device: torch.device) -> bool:
return (
isinstance(w, torch.Tensor)
and w.dtype is torch.bfloat16
and w.is_cuda
and w.device == device
and w.shape == (_HEAD_DIM,)
and w.is_contiguous()
)
def _is_rope(rope: torch.Tensor, seq: int, device: torch.device) -> bool:
return (
isinstance(rope, torch.Tensor)
and rope.dtype is torch.complex64
and rope.device == device
and rope.shape == (seq, _HEAD_DIM // 2)
and rope.is_contiguous()
)
def _cuda_platform(t: torch.Tensor) -> bool:
return t.is_cuda and torch.version.hip is None and not torch.compiler.is_compiling()
def _rope_real(rope: torch.Tensor) -> torch.Tensor:
return torch.view_as_real(rope).view(rope.shape[0], _HEAD_DIM)
def can_use_qknorm_complex_rope_cuda(
x: torch.Tensor, weight: torch.Tensor, rope: torch.Tensor
) -> bool:
"""Contiguous ``[B, S, H, 128]`` bf16 Q or K with its ``(S, 64)`` complex64 table."""
return (
_cuda_platform(x)
and _is_rows(x, x.device)
and x.is_contiguous()
and _is_weight(weight, x.device)
and _is_rope(rope, x.shape[1], x.device)
)
def can_use_qknorm_complex_rope_pack(
q: torch.Tensor,
k_out: torch.Tensor,
v_out: torch.Tensor,
q_weight: torch.Tensor,
k_weight: torch.Tensor,
rope: torch.Tensor,
k_prefix: torch.Tensor,
v_prefix: torch.Tensor,
k_src: Optional[torch.Tensor],
v_src: Optional[torch.Tensor],
) -> bool:
"""Shapes for :func:`qknorm_complex_rope_pack_`; ``None`` sources mean "already in the buffer"."""
if not (_cuda_platform(q) and _is_rows(q, q.device)):
return False
device = q.device
batch, seq, heads, _ = q.shape
if not (
k_prefix.dim() == 4
and k_prefix.shape[0] == batch
and k_prefix.shape[1] > 0
and k_prefix.shape[2] == heads
and k_prefix.shape[3] == _HEAD_DIM
and k_prefix.is_contiguous()
and v_prefix.shape == k_prefix.shape
and v_prefix.is_contiguous()
and all(
t.dtype is torch.bfloat16 and t.device == device
for t in (k_prefix, v_prefix)
)
):
return False
prefix = k_prefix.shape[1]
return (
_is_rows(k_out, device, prefix + seq)
and _is_rows(v_out, device, prefix + seq)
and k_out.shape[0] == batch
and v_out.shape[0] == batch
and k_out.shape[2] == heads
and v_out.shape[2] == heads
and (k_src is None or (_is_rows(k_src, device, seq) and k_src.shape == q.shape))
and (v_src is None or (_is_rows(v_src, device, seq) and v_src.shape == q.shape))
and _is_weight(q_weight, device)
and _is_weight(k_weight, device)
and _is_rope(rope, seq, device)
)
def _fake_norm_rope(x, weight, rope, eps):
return torch.empty_like(x)
@register_custom_op(
op_name="qknorm_complex_rope_cuda",
mutates_args=[],
fake_impl=_fake_norm_rope,
)
def qknorm_complex_rope_cuda(
x: torch.Tensor, weight: torch.Tensor, rope: torch.Tensor, eps: float
) -> torch.Tensor:
"""``rope(rmsnorm(x))`` for one contiguous ``[B, S, H, 128]`` tensor, out of place."""
out = torch.empty_like(x)
module = _jit_module(_fuse_real_sin(x.device), False)
with torch.cuda.device(x.device):
module.norm_rope(x, out, weight, _rope_real(rope), float(eps))
return out
@register_custom_op(
op_name="qknorm_complex_rope_pack_",
mutates_args=["q", "k_out", "v_out"],
)
def qknorm_complex_rope_pack_(
q: torch.Tensor,
k_out: torch.Tensor,
v_out: torch.Tensor,
q_weight: torch.Tensor,
k_weight: torch.Tensor,
rope: torch.Tensor,
k_prefix: torch.Tensor,
v_prefix: torch.Tensor,
k_src: Optional[torch.Tensor],
v_src: Optional[torch.Tensor],
eps: float,
) -> None:
"""Normalize/rotate Q in place and K into ``k_out[:, P:]``; fill the prefix rows of ``k_out``/``v_out``.
``k_src=None`` normalizes ``k_out[:, P:]`` in place (the projection wrote
the raw K there). ``v_src=None`` leaves ``v_out[:, P:]`` untouched (the
projection wrote V there); otherwise ``v_src`` is copied into it.
"""
prefix = k_prefix.shape[1]
token_rows = k_out[:, prefix:]
copy_v = v_src is not None
module = _jit_module(_fuse_real_sin(q.device), copy_v)
with torch.cuda.device(q.device):
module.norm_rope_pack(
q,
q_weight,
token_rows if k_src is None else k_src,
k_out,
k_weight,
v_out,
v_src if copy_v else token_rows,
_rope_real(rope),
k_prefix,
v_prefix,
float(eps),
)
__all__ = [
"can_use_qknorm_complex_rope_cuda",
"can_use_qknorm_complex_rope_pack",
"qknorm_complex_rope_cuda",
"qknorm_complex_rope_pack_",
]
@@ -49,6 +49,8 @@ framework-specific optimization workflow.
- `python/sglang/multimodal_gen/runtime/models/vaes/flux2_vae_cuda_opt.py`
- `python/sglang/multimodal_gen/runtime/models/vaes/wan_vae_cuda_opt.py`
- `python/sglang/multimodal_gen/runtime/models/vaes/autoencoder_kl_qwenimage.py`
- `python/sglang/multimodal_gen/runtime/models/dits/qwen_image21.py`
- `python/sglang/kernels/ops/diffusion/rope/qknorm_complex_rope_jit.py`
- `python/sglang/multimodal_gen/runtime/breakable_cuda_graph/runner.py`
- `test/registered/kernels/ops/diffusion/test_modulate.py`
- `test/registered/kernels/ops/diffusion/test_norm.py`
@@ -286,6 +288,33 @@ framework-specific optimization workflow.
contiguity, group layout, and platform before proposing another router
kernel.
17. Qwen-Image 2.1 attention prep
- Kernels: `qknorm_complex_rope_cuda`, `qknorm_complex_rope_pack_`.
- Locations: `rope/qknorm_complex_rope_jit.py`,
`csrc/diffusion/qknorm_complex_rope.cuh`, and
`runtime/models/dits/qwen_image21.py`.
- Default-on and bit-exact (`BitExactFusionGate`, first call verified against
the eager chain): `QwenImage21Attention` projects Q, K and V with one GEMM
over a packed `[3C, C]` weight view straight into a `[1, P+S, 3C]` buffer
whose rows `[P:]` are the target tokens; one CUDA launch then normalizes Q
and K in place (the `RMSNorm(cast_x_before_out_mul=True)` reduction order),
applies the complex64 RoPE and copies the prefix K/V into rows `[:P]`, so V
is never copied. `apply_qk_norm_rope` uses the same kernel out of place, and
the single-element `torch.cat` on the attention output is gone. Unsupported
layouts (B > 1, SP > 1, LoRA-wrapped or quantized projections,
head_dim != 128) and first-sight mismatches keep the Triton
`qknorm_complex_rope` / `qknorm_complex_rope_kv` chain.
- Validation: `test/registered/kernels/ops/diffusion/test_qknorm_complex_rope.py`,
the `test_qwen21_cuda_qk_rope_*` cases in `test_model_fast_paths.py`, and
`python/sglang/multimodal_gen/test/unit/test_qwen_image21_cuda.py`.
- Workflow rule: if a Qwen-Image 2.1 trace still shows
`_qknorm_complex_rope_onepass_kernel` / `_qknorm_complex_rope_kv_kernel` or
three separate Q/K/V GEMMs per block, check the gate logs for a first-sight
mismatch and the projection layer type before proposing another kernel. The
remaining non-GEMM/attention time per block is the SiLU-mul and the
residual-gate + LayerNorm chain, both measured at the DRAM roofline; a fused
gated LayerNorm was tried and was slower than the two-kernel chain.
**Faster CUDA Kernel Usage Points**
1. sgl-kernel RMSNorm and fused add RMSNorm
@@ -11,10 +11,14 @@ from sglang.kernels.ops.diffusion import (
can_use_fused_complex_rope,
can_use_fused_layernorm_modulate,
can_use_fused_silu_mul,
can_use_qknorm_complex_rope_cuda,
can_use_qknorm_complex_rope_pack,
can_use_rmsnorm_preserve_reduction,
fused_complex_rope,
fused_layernorm_modulate,
fused_silu_mul_bitexact,
qknorm_complex_rope_cuda,
qknorm_complex_rope_pack_,
residual_gate_add,
rmsnorm_preserve_reduction,
tensors_equal,
@@ -41,6 +45,7 @@ from sglang.multimodal_gen.runtime.layers.attention import LocalAttention, USPAt
from sglang.multimodal_gen.runtime.layers.linear import (
ColumnParallelLinear,
RowParallelLinear,
UnquantizedLinearMethod,
)
from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import (
LayerwiseOffloadableModuleMixin,
@@ -55,6 +60,18 @@ _ROPE_FUSION = BitExactFusionGate("Qwen-Image 2.1 complex RoPE")
_SILU_MUL_FUSION = BitExactFusionGate("Qwen-Image 2.1 SiLU-mul")
_QK_ROPE_FUSION = BitExactFusionGate("Qwen-Image 2.1 Q/K RMSNorm + complex RoPE")
_KV_ROPE_FUSION = BitExactFusionGate("Qwen-Image 2.1 K RMSNorm + RoPE + KV packing")
_QK_ROPE_CUDA_FUSION = BitExactFusionGate(
"Qwen-Image 2.1 CUDA Q/K RMSNorm + complex RoPE"
)
_KV_PACK_CUDA_FUSION = BitExactFusionGate(
"Qwen-Image 2.1 CUDA Q/K RMSNorm + RoPE + KV packing"
)
_KV_PROJECT_INTO_FUSION = BitExactFusionGate(
"Qwen-Image 2.1 K/V projection written into the packed KV buffers"
)
_QKV_PACK_FUSION = BitExactFusionGate(
"Qwen-Image 2.1 packed Q/K/V projection written into the packed QKV buffer"
)
_QK_NORM_FUSION = BitExactFusionGate("Qwen-Image 2.1 Q/K RMSNorm")
_MODULATION_FUSION = BitExactFusionGate("Qwen-Image 2.1 LayerNorm modulation")
@@ -141,6 +158,15 @@ def apply_qk_norm(x, norm):
def apply_qk_norm_rope(x, norm, rope):
fused = None
if (
can_use_qknorm_complex_rope_cuda(x, norm.weight, rope)
and _QK_ROPE_CUDA_FUSION.can_attempt_once()
):
fused = qknorm_complex_rope_cuda(x, norm.weight, rope, norm.variance_epsilon)
if _QK_ROPE_CUDA_FUSION.verified:
return fused
out = apply_rope(apply_qk_norm(x, norm), rope)
return _QK_ROPE_CUDA_FUSION.accept_or_fallback(fused, out, logger=logger)
if (
can_use_qknorm_complex_rope(x, norm.weight, rope)
and _QK_ROPE_FUSION.can_attempt_once()
@@ -154,6 +180,41 @@ def apply_qk_norm_rope(x, norm, rope):
return out
def can_project_into(layer, x):
"""A plain bf16 ColumnParallelLinear whose forward is just F.linear on a resident weight."""
return (
type(layer) is ColumnParallelLinear
and isinstance(layer.quant_method, UnquantizedLinearMethod)
and layer.bias is None
and not layer.gather_output
and not layer._forward_pre_hooks
and not layer._forward_hooks
and layer.weight.is_cuda
and layer.weight.dtype is torch.bfloat16
and layer.weight.is_contiguous()
and x.dtype is torch.bfloat16
and x.is_cuda
and x.is_contiguous()
and not torch.compiler.is_compiling()
)
def project_into(layer, x, out):
"""F.linear(x, layer.weight) written into ``out`` (contiguous rows of a larger buffer).
A contiguous row slice has the same leading dimension as a fresh tensor, so
cuBLAS receives the same call as the module forward would issue.
"""
rows = x.shape[0] * x.shape[1]
torch.mm(x.reshape(rows, x.shape[-1]), layer.weight.t(), out=out.view(rows, -1))
return out
def cat_outputs(outputs, dim=0):
# torch.cat on a one-element list degrades to a full copy of the tensor.
return outputs[0] if len(outputs) == 1 else torch.cat(outputs, dim=dim)
def apply_modulation(x, norm, scale):
fused = None
if (
@@ -294,6 +355,74 @@ class QwenImage21Attention(nn.Module):
self.heads, self.head_dim, supported_attention_backends=backends
)
def pack_qkv_weights(self):
"""Share one [3C, C] storage between to_q/to_k/to_v so one GEMM projects all three.
The parameters keep their names and shapes (state_dict, LoRA merge and
weight updates that copy in place all stay valid); only their storage
moves. No reference to the shared storage is kept: ``packed_qkv_weight``
re-derives the [3C, C] view from the parameters, so offloading them
frees the whole buffer. Layers that are not plain bf16 unquantized
projections are left alone and the packed path stays off.
"""
layers = (self.to_q, self.to_k, self.to_v)
if self.packed_qkv_weight() is not None:
return
if not all(
type(layer) is ColumnParallelLinear
and isinstance(layer.quant_method, UnquantizedLinearMethod)
and layer.bias is None
and not layer.gather_output
and layer.weight.is_cuda
and layer.weight.dtype is torch.bfloat16
and layer.weight.is_contiguous()
and layer.weight.shape == self.to_q.weight.shape
for layer in layers
):
return
packed = torch.cat([layer.weight.data for layer in layers], dim=0)
rows = self.to_q.weight.shape[0]
for index, layer in enumerate(layers):
layer.weight.data = packed[index * rows : (index + 1) * rows]
def packed_qkv_weight(self):
"""The [3C, C] view over to_q/to_k/to_v when they are consecutive slices of one storage, else None."""
q, k, v = self.to_q.weight, self.to_k.weight, self.to_v.weight
rows, cols = q.shape
if not (
q.is_cuda
and q.dtype is torch.bfloat16
and k.dtype is q.dtype
and v.dtype is q.dtype
and k.shape == q.shape
and v.shape == q.shape
and q.is_contiguous()
and k.is_contiguous()
and v.is_contiguous()
):
return None
base = q.untyped_storage().data_ptr()
if not (
k.untyped_storage().data_ptr() == base
and v.untyped_storage().data_ptr() == base
):
return None
stride = rows * cols
if not (
k.storage_offset() == q.storage_offset() + stride
and v.storage_offset() == k.storage_offset() + stride
):
return None
return q.as_strided((3 * rows, cols), (cols, 1))
def _apply(self, fn, recurse=True):
# .to() and CPU offload rebuild every parameter in its own storage;
# re-share them once the weights are back on the GPU so the packed
# projection survives the round trip (a no-op anywhere else).
module = super()._apply(fn, recurse)
self.pack_qkv_weights()
return module
def project_qkv(self, x):
q = self.to_q(x)[0].unflatten(-1, (self.heads, self.head_dim))
k = self.to_k(x)[0].unflatten(-1, (self.heads, self.head_dim))
@@ -308,7 +437,150 @@ class QwenImage21Attention(nn.Module):
v,
)
def attend_sample(self, q, k, v, rope, prefix, prefix_rope, segments, cache):
def _pack_kv_cuda(self, q, k, v, rope, kp, vp, k_out=None, v_out=None):
"""One CUDA launch: Q normed+roped in place, K normed+roped and V packed behind the prefix.
With ``k_out``/``v_out`` the projection already wrote the raw K/V into
rows ``[P:]`` (``k``/``v`` are those views), so K is normalized in place
and V is not copied. Returns ``(q, k_out, v_out)`` or ``None`` when the
layout is unsupported. The first call also runs the eager chain and
keeps its result on a mismatch.
"""
eps = self.norm_k.variance_epsilon
if self.norm_q.variance_epsilon != eps:
return None
prefix = kp.shape[1]
if k_out is None:
k_out = k.new_empty(k.shape[0], prefix + k.shape[1], *k.shape[2:])
v_out = torch.empty_like(k_out)
k_src, v_src = k, v
else:
k_src, v_src = None, None
if not can_use_qknorm_complex_rope_pack(
q,
k_out,
v_out,
self.norm_q.weight,
self.norm_k.weight,
rope,
kp,
vp,
k_src,
v_src,
):
return None
reference = None
if not _KV_PACK_CUDA_FUSION.verified:
reference = (
apply_rope(apply_qk_norm(q, self.norm_q), rope),
torch.cat([kp, apply_rope(apply_qk_norm(k, self.norm_k), rope)], 1),
torch.cat([vp, v], 1),
)
qknorm_complex_rope_pack_(
q,
k_out,
v_out,
self.norm_q.weight,
self.norm_k.weight,
rope,
kp,
vp,
k_src,
v_src,
eps,
)
if reference is None:
return q, k_out, v_out
return _KV_PACK_CUDA_FUSION.accept_or_fallback(
(q, k_out, v_out), reference, equal=tensors_equal, logger=logger
)
def _project_qkv_packed(self, x, layouts, caches):
"""One GEMM writes Q, K and V into a ``[1, P+S, 3C]`` buffer; ``None`` keeps the plain path.
Q/K/V become column views with token stride ``3C`` (heads stay contiguous);
the fused norm + RoPE kernel and SDPA both accept that layout. The first
call compares every slice against the module forwards.
"""
layers = (self.to_q, self.to_k, self.to_v)
packed = self.packed_qkv_weight()
if not (
packed is not None
and x.shape[0] == 1
and len(layouts) == 1
and get_sp_world_size() == 1
and _QKV_PACK_FUSION.can_attempt_once()
and all(can_project_into(layer, x) for layer in layers)
):
return None
cache = caches[0]
prefix = cache["key"].shape[1] if cache else layouts[0]["prefix_rope"].shape[0]
seq = x.shape[1]
rows = packed.shape[0] // 3
buffer = x.new_empty(1, prefix + seq, 3 * rows)
torch.mm(x.view(seq, x.shape[-1]), packed.t(), out=buffer[0, prefix:])
q = buffer[:, prefix:, :rows].view(1, seq, self.heads, self.head_dim)
k_out = buffer[:, :, rows : 2 * rows].view(
1, prefix + seq, self.heads, self.head_dim
)
v_out = buffer[:, :, 2 * rows :].view(
1, prefix + seq, self.heads, self.head_dim
)
k, v = k_out[:, prefix:], v_out[:, prefix:]
if not _QKV_PACK_FUSION.verified:
reference = tuple(
layer(x)[0].unflatten(-1, (self.heads, self.head_dim))
for layer in layers
)
accepted = _QKV_PACK_FUSION.accept_or_fallback(
(q, k, v), reference, equal=tensors_equal, logger=logger
)
if accepted is reference:
return None
return q, k, v, k_out, v_out
def _project_kv_into_buffers(self, x, layouts, caches):
"""Project K/V straight into ``[1, P+S, H, D]`` buffers; ``None`` keeps the plain path."""
if not (
x.shape[0] == 1
and len(layouts) == 1
and get_sp_world_size() == 1
and _KV_PROJECT_INTO_FUSION.can_attempt_once()
and can_project_into(self.to_k, x)
and can_project_into(self.to_v, x)
):
return None
cache = caches[0]
prefix = cache["key"].shape[1] if cache else layouts[0]["prefix_rope"].shape[0]
k_out = x.new_empty(1, prefix + x.shape[1], self.heads, self.head_dim)
v_out = torch.empty_like(k_out)
k = project_into(self.to_k, x, k_out[:, prefix:])
v = project_into(self.to_v, x, v_out[:, prefix:])
if not _KV_PROJECT_INTO_FUSION.verified:
reference = (
self.to_k(x)[0].unflatten(-1, (self.heads, self.head_dim)),
self.to_v(x)[0].unflatten(-1, (self.heads, self.head_dim)),
)
accepted = _KV_PROJECT_INTO_FUSION.accept_or_fallback(
(k, v), reference, equal=tensors_equal, logger=logger
)
if accepted is reference:
return reference[0], reference[1], None, None
return k, v, k_out, v_out
def attend_sample(
self,
q,
k,
v,
rope,
prefix,
prefix_rope,
segments,
cache,
k_out=None,
v_out=None,
):
if cache:
kp, vp = cache["key"], cache["value"]
prefix_output = None
@@ -329,9 +601,13 @@ class QwenImage21Attention(nn.Module):
qp[:, start:end], kp[:, :end], vp[:, :end], attn_mask=mask
)
)
prefix_output = self.to_out[0](torch.cat(outputs, dim=1).flatten(2))[0]
prefix_output = self.to_out[0](cat_outputs(outputs, dim=1).flatten(2))[0]
if cache is not None:
cache.update(key=kp, value=vp)
if get_sp_world_size() == 1 and _KV_PACK_CUDA_FUSION.can_attempt_once():
cuda_packed = self._pack_kv_cuda(q, k, v, rope, kp, vp, k_out, v_out)
if cuda_packed is not None:
return self.target_attn(*cuda_packed), prefix_output
q = apply_qk_norm_rope(q, self.norm_q, rope)
packed = None
if (
@@ -362,7 +638,17 @@ class QwenImage21Attention(nn.Module):
def forward(self, x, ropes, prefixes, layouts, caches):
# batch target projections while retaining each sample's unpadded prefix
q, k, v = self.project_qkv(x)
packed = self._project_qkv_packed(x, layouts, caches)
if packed is not None:
q, k, v, k_out, v_out = packed
else:
buffers = self._project_kv_into_buffers(x, layouts, caches)
if buffers is None:
q, k, v = self.project_qkv(x)
k_out = v_out = None
else:
q = self.to_q(x)[0].unflatten(-1, (self.heads, self.head_dim))
k, v, k_out, v_out = buffers
outputs, prefix_outputs = [], []
for sample, layout in enumerate(layouts):
out, prefix_out = self.attend_sample(
@@ -374,10 +660,12 @@ class QwenImage21Attention(nn.Module):
layout["prefix_rope"],
layout["segments"],
caches[sample],
k_out=k_out,
v_out=v_out,
)
outputs.append(out)
prefix_outputs.append(prefix_out)
return self.to_out[0](torch.cat(outputs).flatten(2))[0], prefix_outputs
return self.to_out[0](cat_outputs(outputs).flatten(2))[0], prefix_outputs
class QwenImage21TransformerBlock(nn.Module):
@@ -498,6 +786,11 @@ class QwenImage21Transformer2DModel(CachableDiT, LayerwiseOffloadableModuleMixin
self.norm_out = QwenImage21OutputNorm(ac.hidden_size, ac.eps)
self.proj_out = nn.Linear(ac.hidden_size, ac.out_channels, bias=False)
def post_load_weights(self):
super().post_load_weights()
for block in self.transformer_blocks:
block.attn.pack_qkv_weights()
def prepare_modulation(self, temb):
# All blocks share these gates. Preserve the native tanh and its dtype,
# but compute it once per timestep instead of once per block.
@@ -371,3 +371,183 @@ def test_silu_fusion_mismatch_restores_eager(bf16_model, monkeypatch):
torch.testing.assert_close(mlp(x), expected, atol=0, rtol=0)
assert gate.disabled and not gate.verified
torch.testing.assert_close(mlp(x), expected, atol=0, rtol=0)
@pytest.fixture
def bf16_model_hd128(model):
# head_dim 128 is what the CUDA Q/K norm + RoPE + KV packing kernel addresses;
# the shared fixture's head_dim 32 never reaches it.
config = QwenImage21DitConfig(
arch_config=QwenImage21ArchConfig(
in_channels=4,
out_channels=4,
num_layers=2,
num_attention_heads=2,
attention_head_dim=128,
context_in_dim=16,
mlp_ratio=2,
axes_dims_rope=(16, 56, 56),
)
)
torch.manual_seed(7)
result = QwenImage21Transformer2DModel(config, {}).cuda().bfloat16().eval()
for name, param in result.named_parameters():
torch.nn.init.normal_(param, std=0.02)
if name.endswith(("norm_q.weight", "norm_k.weight")):
torch.nn.init.normal_(param, mean=1.0, std=0.1)
result.post_load_weights()
return result
def inputs_hd128(seed, edit):
torch.manual_seed(seed)
slots = [False] * 3 + ([True, False, False] if edit else [])
shapes = ([(1, 2, 4)] if edit else []) + [(1, 4, 4)]
kwargs = dict(
hidden_states=torch.randn(1, 16, 4, device="cuda"),
encoder_hidden_states=torch.randn(1, len(slots), 16, device="cuda"),
condition_latents=torch.randn(1, 8, 4, device="cuda") if edit else None,
layouts=[build_layout(slots, shapes, (16, 56, 56), "cuda")],
prefix_caches=[[{} for _ in range(2)]],
timestep=torch.tensor([700.0], device="cuda"),
)
for key in (
"hidden_states",
"encoder_hidden_states",
"condition_latents",
"timestep",
):
if kwargs[key] is not None:
kwargs[key] = kwargs[key].bfloat16()
return kwargs
@pytest.mark.parametrize("edit", [False, True])
@torch.no_grad()
def test_cuda_qk_rope_pack_matches_eager_prefill_and_cached_steps(
bf16_model_hd128, edit, monkeypatch
):
# The CUDA Q/K norm + RoPE + KV packing path must reproduce the Triton/eager
# chain bit for bit on the prefill step, on cached steps and under BCG replay.
actual_model = bf16_model_hd128
kwargs = inputs_hd128(11, edit)
reference_kwargs = deepcopy(kwargs)
expected = []
disabled = BitExactFusionGate("reference")
disabled.disable()
with monkeypatch.context() as reference, set_forward_context(None, None):
reference.setattr(model_module, "_QK_ROPE_CUDA_FUSION", disabled)
reference.setattr(model_module, "_KV_PACK_CUDA_FUSION", disabled)
reference.setattr(model_module, "_KV_PROJECT_INTO_FUSION", disabled)
reference.setattr(model_module, "_QKV_PACK_FUSION", disabled)
for timestep in (700, 300, 10):
reference_kwargs["timestep"].fill_(timestep)
expected.append(actual_model(**reference_kwargs))
qk_gate = BitExactFusionGate("test CUDA Q/K norm + RoPE")
kv_gate = BitExactFusionGate("test CUDA KV packing")
project_gate = BitExactFusionGate("test K/V projection into buffers")
qkv_gate = BitExactFusionGate("test packed Q/K/V projection")
monkeypatch.setattr(model_module, "_QK_ROPE_CUDA_FUSION", qk_gate)
monkeypatch.setattr(model_module, "_KV_PACK_CUDA_FUSION", kv_gate)
monkeypatch.setattr(model_module, "_KV_PROJECT_INTO_FUSION", project_gate)
monkeypatch.setattr(model_module, "_QKV_PACK_FUSION", qkv_gate)
with set_forward_context(None, None):
for timestep, output in zip((700, 300, 10), expected, strict=True):
kwargs["timestep"].fill_(timestep)
torch.testing.assert_close(actual_model(**kwargs), output, atol=0, rtol=0)
# The CUDA kernels are exact by construction and must have engaged.
for gate in (qk_gate, kv_gate):
assert gate.verified and not gate.disabled, gate.name
# The packed and the direct-write projections are GEMM re-plumbings whose
# first-sight compare depends on cuBLAS picking the same kernel for both
# shapes; where it does not, the gate declines and the next tier takes over
# (the outputs above are bit-exact either way). When the packed projection
# verified, it supersedes the per-layer direct write, which never attempts.
assert qkv_gate.verified or qkv_gate.disabled
if qkv_gate.verified:
assert not project_gate.verified and not project_gate.disabled
else:
assert project_gate.verified or project_gate.disabled
for actual, reference in zip(
kwargs["prefix_caches"][0], reference_kwargs["prefix_caches"][0], strict=True
):
for key in ("key", "value"):
torch.testing.assert_close(actual[key], reference[key], atol=0, rtol=0)
assert (
actual[key].untyped_storage().nbytes()
== actual[key].numel() * actual[key].element_size()
)
runner = DiffusionBreakableCudaGraphRunner(actual_model, torch.device("cuda"))
try:
with set_forward_context(None, None):
assert runner.capture(**kwargs)
kwargs["hidden_states"].add_(0.1)
expected = actual_model(**kwargs)
torch.testing.assert_close(runner(**kwargs), expected, atol=0, rtol=0)
finally:
runner.reset()
@torch.no_grad()
def test_cuda_kv_pack_mismatch_restores_reference(bf16_model_hd128, monkeypatch):
# A kernel that disagrees with the reference must disable itself and leave
# the model output on the Triton/eager chain.
kwargs = inputs_hd128(12, False)
reference_kwargs = deepcopy(kwargs)
disabled = BitExactFusionGate("reference")
disabled.disable()
with monkeypatch.context() as reference, set_forward_context(None, None):
reference.setattr(model_module, "_QK_ROPE_CUDA_FUSION", disabled)
reference.setattr(model_module, "_KV_PACK_CUDA_FUSION", disabled)
reference.setattr(model_module, "_QKV_PACK_FUSION", disabled)
expected = bf16_model_hd128(**reference_kwargs)
kv_gate = BitExactFusionGate("test mismatched KV packing")
monkeypatch.setattr(model_module, "_KV_PACK_CUDA_FUSION", kv_gate)
monkeypatch.setattr(model_module, "_QK_ROPE_CUDA_FUSION", disabled)
def corrupt(q, k_out, v_out, *args):
v_out.zero_()
monkeypatch.setattr(model_module, "qknorm_complex_rope_pack_", corrupt)
with set_forward_context(None, None):
torch.testing.assert_close(bf16_model_hd128(**kwargs), expected, atol=0, rtol=0)
assert kv_gate.disabled and not kv_gate.verified
@torch.no_grad()
def test_packed_qkv_weights_share_storage_and_survive_in_place_updates(
bf16_model_hd128,
):
# pack_qkv_weights must keep parameter names/values and alias the packed buffer,
# so a merge-mode LoRA delta written into to_q.weight is what the packed GEMM sees.
attn = bf16_model_hd128.transformer_blocks[0].attn
packed = attn.packed_qkv_weight()
assert packed is not None
rows = attn.to_q.weight.shape[0]
assert torch.equal(packed[:rows], attn.to_q.weight)
assert torch.equal(packed[rows : 2 * rows], attn.to_k.weight)
assert torch.equal(packed[2 * rows :], attn.to_v.weight)
keys = set(bf16_model_hd128.state_dict().keys())
assert "transformer_blocks.0.attn.to_q.weight" in keys
assert not any("qkv_weight" in key for key in keys)
before = packed[:rows].clone()
attn.to_q.weight.add_(1.0)
assert torch.equal(attn.packed_qkv_weight()[:rows], before + 1.0)
attn.to_q.weight.sub_(1.0)
# An offload round trip must not pin the shared CUDA storage: nothing on the
# module keeps a device tensor once the parameters moved, and the shared
# views come back with the weights.
attn.to("cpu")
assert attn.packed_qkv_weight() is None
assert not any(
isinstance(value, torch.Tensor) and value.is_cuda
for value in vars(attn).values()
)
attn.to("cuda")
packed = attn.packed_qkv_weight()
assert packed is not None
assert torch.equal(packed[:rows], attn.to_q.weight)
assert torch.equal(packed[2 * rows :], attn.to_v.weight)