[diffusion] Qwen-Image 2.1: fuse Q/K RMSNorm + RoPE + KV packing into one CUDA kernel and project Q/K/V with one packed GEMM (#41339)
This commit is contained in:
@@ -0,0 +1,344 @@
|
||||
// Bit-exact 128-wide RMSNorm + complex RoPE for Qwen-Image-2.1 attention prep.
|
||||
//
|
||||
// Eager chain reproduced for bf16 activations:
|
||||
// RMSNorm(cast_x_before_out_mul=True): var = mean(float(x)^2) in aten's
|
||||
// vectorized 128-wide fp32 mean order (lane t owns elements 4t..4t+3, sums
|
||||
// them left to right, then a 32-lane shuffle tree with offsets 16,8,4,2,1),
|
||||
// inv = rsqrtf(var + eps), n = bf16(x * inv), o = bf16(n * w);
|
||||
// complex RoPE on interleaved pairs: out = view_as_real(complex(o) * rope),
|
||||
// with the FMA orientation PyTorch's complex64 multiply uses on this GPU
|
||||
// (kFuseRealSin, probed on the Python side).
|
||||
// One launch handles the Q rows and the K rows (source -> destination, in place
|
||||
// when they alias), copies the cached prefix K/V into rows [0:P] of the
|
||||
// [B, P+S, H, 128] K/V buffers and, with kCopyV, the raw V rows into [P:P+S].
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <sgl_kernel/tensor.h>
|
||||
#include <sgl_kernel/utils.h>
|
||||
|
||||
#include <sgl_kernel/type.cuh>
|
||||
#include <sgl_kernel/utils.cuh>
|
||||
#include <sgl_kernel/vec.cuh>
|
||||
|
||||
#include <algorithm>
|
||||
#include <cstdint>
|
||||
|
||||
namespace sglang {
|
||||
|
||||
namespace qknorm_complex_rope {
|
||||
|
||||
constexpr int kHeadDim = 128;
|
||||
constexpr int kLaneElems = kHeadDim / device::kWarpThreads; // 4 bf16 per lane
|
||||
constexpr uint32_t kThreads = 256;
|
||||
constexpr uint32_t kWarpsPerBlock = kThreads / device::kWarpThreads;
|
||||
constexpr uint32_t kMaxBlocks = 65535;
|
||||
constexpr int64_t kVecAlignBytes = 8;
|
||||
|
||||
static_assert(kLaneElems == 4, "the aten mean replica assumes four elements per lane");
|
||||
|
||||
using RowVec = device::AlignedVector<bf16_t, kLaneElems>; // 8 bytes
|
||||
using RopeVec = device::AlignedVector<fp32_t, kLaneElems>; // cos0 sin0 cos1 sin1
|
||||
|
||||
/// Strided [B, T, H, 128] tensor: element (b, t, h, d) lives at ptr + b*batch_stride + t*token_stride + h*128 + d.
|
||||
struct Rows {
|
||||
const bf16_t* ptr;
|
||||
int64_t batch_stride;
|
||||
int64_t token_stride;
|
||||
};
|
||||
|
||||
struct MutableRows {
|
||||
bf16_t* ptr;
|
||||
int64_t batch_stride;
|
||||
int64_t token_stride;
|
||||
};
|
||||
|
||||
struct Params {
|
||||
Rows q_src;
|
||||
MutableRows q_dst;
|
||||
Rows k_src;
|
||||
MutableRows k_dst; // [B, P+S, H, 128]; token rows land at P + s
|
||||
MutableRows v_dst; // [B, P+S, H, 128]
|
||||
Rows v_src; // read only with kCopyV
|
||||
const bf16_t* q_weight;
|
||||
const bf16_t* k_weight;
|
||||
const bf16_t* k_prefix; // contiguous [B, P, H, 128]
|
||||
const bf16_t* v_prefix; // contiguous [B, P, H, 128]
|
||||
const fp32_t* rope; // [S, 128] = view_as_real of the complex64 [S, 64] table
|
||||
uint32_t batch;
|
||||
uint32_t seq;
|
||||
uint32_t heads;
|
||||
uint32_t prefix;
|
||||
uint32_t k_rows_enabled; // 0 for the single-tensor entry point
|
||||
float eps;
|
||||
};
|
||||
|
||||
SGL_DEVICE const bf16_t* row_ptr(const Rows& rows, int64_t b, int64_t t, int64_t h) {
|
||||
return rows.ptr + b * rows.batch_stride + t * rows.token_stride + h * kHeadDim;
|
||||
}
|
||||
|
||||
SGL_DEVICE bf16_t* row_ptr(const MutableRows& rows, int64_t b, int64_t t, int64_t h) {
|
||||
return rows.ptr + b * rows.batch_stride + t * rows.token_stride + h * kHeadDim;
|
||||
}
|
||||
|
||||
/// \brief Normalize and rotate one 128-wide row; every lane owns 4 consecutive elements.
|
||||
template <bool kFuseRealSin>
|
||||
SGL_DEVICE void norm_rope_row(
|
||||
const bf16_t* __restrict__ src,
|
||||
bf16_t* __restrict__ dst,
|
||||
const bf16_t* __restrict__ weight,
|
||||
const fp32_t* __restrict__ rope_row,
|
||||
float eps,
|
||||
int lane) {
|
||||
RowVec x;
|
||||
RowVec w;
|
||||
RopeVec r;
|
||||
x.load(src + lane * kLaneElems);
|
||||
w.load(weight + lane * kLaneElems);
|
||||
r.load(rope_row + lane * kLaneElems);
|
||||
|
||||
float value[kLaneElems];
|
||||
#pragma unroll
|
||||
for (int i = 0; i < kLaneElems; ++i) {
|
||||
value[i] = device::cast<fp32_t>(x[i]);
|
||||
}
|
||||
// aten vectorized mean: four squares combined left to right ...
|
||||
float part = __fmul_rn(value[0], value[0]);
|
||||
part = __fadd_rn(part, __fmul_rn(value[1], value[1]));
|
||||
part = __fadd_rn(part, __fmul_rn(value[2], value[2]));
|
||||
part = __fadd_rn(part, __fmul_rn(value[3], value[3]));
|
||||
// ... then the 32-lane tree with decreasing offsets (matches shfl_down 16..1 for lane 0).
|
||||
#pragma unroll
|
||||
for (int offset = device::kWarpThreads / 2; offset > 0; offset >>= 1) {
|
||||
part = __fadd_rn(part, __shfl_xor_sync(0xffffffffu, part, offset));
|
||||
}
|
||||
const float variance = __fmul_rn(part, 1.0f / kHeadDim);
|
||||
const float inv = rsqrtf(__fadd_rn(variance, eps));
|
||||
|
||||
// cast_x_before_out_mul: round after the normalize, then again after the weight.
|
||||
float out_f32[kLaneElems];
|
||||
#pragma unroll
|
||||
for (int i = 0; i < kLaneElems; ++i) {
|
||||
const float normalized = device::cast<fp32_t>(device::cast<bf16_t>(__fmul_rn(value[i], inv)));
|
||||
out_f32[i] = device::cast<fp32_t>(device::cast<bf16_t>(__fmul_rn(normalized, device::cast<fp32_t>(w[i]))));
|
||||
}
|
||||
|
||||
RowVec out;
|
||||
#pragma unroll
|
||||
for (int p = 0; p < kLaneElems / 2; ++p) {
|
||||
const float re = out_f32[2 * p];
|
||||
const float im = out_f32[2 * p + 1];
|
||||
const float cos = r[2 * p];
|
||||
const float sin = r[2 * p + 1];
|
||||
const float out_re = __fmaf_rn(re, cos, -__fmul_rn(im, sin));
|
||||
float out_im;
|
||||
if constexpr (kFuseRealSin) {
|
||||
out_im = __fmaf_rn(re, sin, __fmul_rn(im, cos));
|
||||
} else {
|
||||
out_im = __fmaf_rn(im, cos, __fmul_rn(re, sin));
|
||||
}
|
||||
out[2 * p] = device::cast<bf16_t>(out_re);
|
||||
out[2 * p + 1] = device::cast<bf16_t>(out_im);
|
||||
}
|
||||
out.store(dst + lane * kLaneElems);
|
||||
}
|
||||
|
||||
SGL_DEVICE void copy_row(const bf16_t* __restrict__ src, bf16_t* __restrict__ dst, int lane) {
|
||||
RowVec v;
|
||||
v.load(src + lane * kLaneElems);
|
||||
v.store(dst + lane * kLaneElems);
|
||||
}
|
||||
|
||||
/// Work list: Q rows, K rows, prefix-K copies, prefix-V copies, then (kCopyV) V copies. One warp per row.
|
||||
template <bool kFuseRealSin, bool kCopyV>
|
||||
__global__ void kernel(const Params __grid_constant__ params) {
|
||||
const int lane = threadIdx.x & (device::kWarpThreads - 1);
|
||||
const uint32_t warp = threadIdx.x >> 5;
|
||||
const int64_t heads = params.heads;
|
||||
const int64_t seq = params.seq;
|
||||
const int64_t prefix = params.prefix;
|
||||
const int64_t rows_q = static_cast<int64_t>(params.batch) * seq * heads;
|
||||
const int64_t rows_k = params.k_rows_enabled ? rows_q : 0;
|
||||
const int64_t rows_p = params.k_rows_enabled ? static_cast<int64_t>(params.batch) * prefix * heads : 0;
|
||||
const int64_t rows_v = kCopyV ? rows_q : 0;
|
||||
const int64_t total = rows_q + rows_k + 2 * rows_p + rows_v;
|
||||
const int64_t stride = static_cast<int64_t>(gridDim.x) * kWarpsPerBlock;
|
||||
|
||||
for (int64_t row = static_cast<int64_t>(blockIdx.x) * kWarpsPerBlock + warp; row < total; row += stride) {
|
||||
int64_t r = row;
|
||||
if (r < rows_q) {
|
||||
const int64_t h = r % heads, t = r / heads, s = t % seq, b = t / seq;
|
||||
norm_rope_row<kFuseRealSin>(
|
||||
row_ptr(params.q_src, b, s, h),
|
||||
row_ptr(params.q_dst, b, s, h),
|
||||
params.q_weight,
|
||||
params.rope + s * kHeadDim,
|
||||
params.eps,
|
||||
lane);
|
||||
continue;
|
||||
}
|
||||
r -= rows_q;
|
||||
if (r < rows_k) {
|
||||
const int64_t h = r % heads, t = r / heads, s = t % seq, b = t / seq;
|
||||
norm_rope_row<kFuseRealSin>(
|
||||
row_ptr(params.k_src, b, s, h),
|
||||
row_ptr(params.k_dst, b, prefix + s, h),
|
||||
params.k_weight,
|
||||
params.rope + s * kHeadDim,
|
||||
params.eps,
|
||||
lane);
|
||||
continue;
|
||||
}
|
||||
r -= rows_k;
|
||||
if (r < rows_p) {
|
||||
const int64_t h = r % heads, t = r / heads, p = t % prefix, b = t / prefix;
|
||||
copy_row(params.k_prefix + ((b * prefix + p) * heads + h) * kHeadDim, row_ptr(params.k_dst, b, p, h), lane);
|
||||
continue;
|
||||
}
|
||||
r -= rows_p;
|
||||
if (r < rows_p) {
|
||||
const int64_t h = r % heads, t = r / heads, p = t % prefix, b = t / prefix;
|
||||
copy_row(params.v_prefix + ((b * prefix + p) * heads + h) * kHeadDim, row_ptr(params.v_dst, b, p, h), lane);
|
||||
continue;
|
||||
}
|
||||
if constexpr (kCopyV) {
|
||||
r -= rows_p;
|
||||
const int64_t h = r % heads, t = r / heads, s = t % seq, b = t / seq;
|
||||
copy_row(row_ptr(params.v_src, b, s, h), row_ptr(params.v_dst, b, prefix + s, h), lane);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <bool kFuseRealSin, bool kCopyV>
|
||||
struct Kernel {
|
||||
/// \brief Single tensor, out of place: `out = rope(rmsnorm(x))` for a contiguous [B, S, H, 128] bf16 tensor.
|
||||
static void
|
||||
run(tvm::ffi::TensorView x,
|
||||
tvm::ffi::TensorView out,
|
||||
tvm::ffi::TensorView weight,
|
||||
tvm::ffi::TensorView rope,
|
||||
double eps) {
|
||||
using namespace host;
|
||||
auto B = SymbolicSize{"batch"};
|
||||
auto S = SymbolicSize{"seq"};
|
||||
auto H = SymbolicSize{"heads"};
|
||||
auto device = SymbolicDevice{};
|
||||
device.set_options<kDLCUDA>();
|
||||
TensorMatcher({B, S, H, kHeadDim})
|
||||
.with_dtype<bf16_t>()
|
||||
.with_device(device)
|
||||
.ensure_alignment(kVecAlignBytes)
|
||||
.verify(x)
|
||||
.verify(out);
|
||||
TensorMatcher({kHeadDim}).with_dtype<bf16_t>().with_device(device).verify(weight);
|
||||
TensorMatcher({S, kHeadDim}).with_dtype<fp32_t>().with_device(device).ensure_alignment(16).verify(rope);
|
||||
const int64_t batch = B.unwrap(), seq = S.unwrap(), heads = H.unwrap();
|
||||
if (batch == 0 || seq == 0 || heads == 0) return;
|
||||
const int64_t token_stride = heads * kHeadDim;
|
||||
Params params{};
|
||||
params.q_src = Rows{static_cast<const bf16_t*>(x.data_ptr()), seq * token_stride, token_stride};
|
||||
params.q_dst = MutableRows{static_cast<bf16_t*>(out.data_ptr()), seq * token_stride, token_stride};
|
||||
params.q_weight = static_cast<const bf16_t*>(weight.data_ptr());
|
||||
params.k_weight = params.q_weight;
|
||||
params.rope = static_cast<const fp32_t*>(rope.data_ptr());
|
||||
params.batch = static_cast<uint32_t>(batch);
|
||||
params.seq = static_cast<uint32_t>(seq);
|
||||
params.heads = static_cast<uint32_t>(heads);
|
||||
params.prefix = 0;
|
||||
params.k_rows_enabled = 0;
|
||||
params.eps = static_cast<float>(eps);
|
||||
launch(params, batch * seq * heads, device.unwrap());
|
||||
}
|
||||
|
||||
/// \brief Q in place, K source -> rows [P:] of k_out, prefix K/V -> rows [0:P] of k_out/v_out,
|
||||
/// raw V -> rows [P:] of v_out when kCopyV (v_src is ignored otherwise).
|
||||
static void run_pack(
|
||||
tvm::ffi::TensorView q,
|
||||
tvm::ffi::TensorView q_weight,
|
||||
tvm::ffi::TensorView k_src,
|
||||
tvm::ffi::TensorView k_out,
|
||||
tvm::ffi::TensorView k_weight,
|
||||
tvm::ffi::TensorView v_out,
|
||||
tvm::ffi::TensorView v_src,
|
||||
tvm::ffi::TensorView rope,
|
||||
tvm::ffi::TensorView k_prefix,
|
||||
tvm::ffi::TensorView v_prefix,
|
||||
double eps) {
|
||||
using namespace host;
|
||||
auto B = SymbolicSize{"batch"};
|
||||
auto S = SymbolicSize{"seq"};
|
||||
auto H = SymbolicSize{"heads"};
|
||||
auto P = SymbolicSize{"prefix"};
|
||||
auto PS = SymbolicSize{"prefix_plus_seq"};
|
||||
auto Bq = SymbolicSize{"q_batch_stride"}, Tq = SymbolicSize{"q_token_stride"};
|
||||
auto Bk = SymbolicSize{"k_src_batch_stride"}, Tk = SymbolicSize{"k_src_token_stride"};
|
||||
auto Bko = SymbolicSize{"k_out_batch_stride"}, Tko = SymbolicSize{"k_out_token_stride"};
|
||||
auto Bvo = SymbolicSize{"v_out_batch_stride"}, Tvo = SymbolicSize{"v_out_token_stride"};
|
||||
auto Bv = SymbolicSize{"v_src_batch_stride"}, Tv = SymbolicSize{"v_src_token_stride"};
|
||||
auto device = SymbolicDevice{};
|
||||
device.set_options<kDLCUDA>();
|
||||
// TensorMatcher is not copyable, so verify inside the lambda; the member
|
||||
// template call on a dependent expression needs `template`.
|
||||
const auto verify_rows = [&](const tvm::ffi::TensorView& t, auto& rows, auto& batch_stride, auto& token_stride) {
|
||||
TensorMatcher({B, rows, H, kHeadDim})
|
||||
.with_strides({batch_stride, token_stride, kHeadDim, 1})
|
||||
.template with_dtype<bf16_t>()
|
||||
.with_device(device)
|
||||
.ensure_alignment(kVecAlignBytes)
|
||||
.verify(t);
|
||||
};
|
||||
verify_rows(q, S, Bq, Tq);
|
||||
verify_rows(k_src, S, Bk, Tk);
|
||||
verify_rows(k_out, PS, Bko, Tko);
|
||||
verify_rows(v_out, PS, Bvo, Tvo);
|
||||
if constexpr (kCopyV) {
|
||||
verify_rows(v_src, S, Bv, Tv);
|
||||
}
|
||||
TensorMatcher({B, P, H, kHeadDim})
|
||||
.with_dtype<bf16_t>()
|
||||
.with_device(device)
|
||||
.ensure_alignment(kVecAlignBytes)
|
||||
.verify(k_prefix)
|
||||
.verify(v_prefix);
|
||||
TensorMatcher({kHeadDim}).with_dtype<bf16_t>().with_device(device).verify(q_weight).verify(k_weight);
|
||||
TensorMatcher({S, kHeadDim}).with_dtype<fp32_t>().with_device(device).ensure_alignment(16).verify(rope);
|
||||
const int64_t batch = B.unwrap(), seq = S.unwrap(), heads = H.unwrap(), prefix = P.unwrap();
|
||||
CHECK_HOST(PS.unwrap() == prefix + seq) << "k_out/v_out must hold prefix + seq tokens";
|
||||
if (batch == 0 || seq == 0 || heads == 0) return;
|
||||
|
||||
Params params{};
|
||||
params.q_src = Rows{static_cast<const bf16_t*>(q.data_ptr()), Bq.unwrap(), Tq.unwrap()};
|
||||
params.q_dst = MutableRows{static_cast<bf16_t*>(q.data_ptr()), Bq.unwrap(), Tq.unwrap()};
|
||||
params.k_src = Rows{static_cast<const bf16_t*>(k_src.data_ptr()), Bk.unwrap(), Tk.unwrap()};
|
||||
params.k_dst = MutableRows{static_cast<bf16_t*>(k_out.data_ptr()), Bko.unwrap(), Tko.unwrap()};
|
||||
params.v_dst = MutableRows{static_cast<bf16_t*>(v_out.data_ptr()), Bvo.unwrap(), Tvo.unwrap()};
|
||||
if constexpr (kCopyV) {
|
||||
params.v_src = Rows{static_cast<const bf16_t*>(v_src.data_ptr()), Bv.unwrap(), Tv.unwrap()};
|
||||
}
|
||||
params.q_weight = static_cast<const bf16_t*>(q_weight.data_ptr());
|
||||
params.k_weight = static_cast<const bf16_t*>(k_weight.data_ptr());
|
||||
params.k_prefix = static_cast<const bf16_t*>(k_prefix.data_ptr());
|
||||
params.v_prefix = static_cast<const bf16_t*>(v_prefix.data_ptr());
|
||||
params.rope = static_cast<const fp32_t*>(rope.data_ptr());
|
||||
params.batch = static_cast<uint32_t>(batch);
|
||||
params.seq = static_cast<uint32_t>(seq);
|
||||
params.heads = static_cast<uint32_t>(heads);
|
||||
params.prefix = static_cast<uint32_t>(prefix);
|
||||
params.k_rows_enabled = 1;
|
||||
params.eps = static_cast<float>(eps);
|
||||
const int64_t rows = batch * heads * (2 * seq + 2 * prefix + (kCopyV ? seq : 0));
|
||||
launch(params, rows, device.unwrap());
|
||||
}
|
||||
|
||||
private:
|
||||
static void launch(const Params& params, int64_t rows, DLDevice device) {
|
||||
using namespace host;
|
||||
const int64_t needed = div_ceil(rows, static_cast<int64_t>(kWarpsPerBlock));
|
||||
const auto blocks = static_cast<uint32_t>(std::min<int64_t>(needed, kMaxBlocks));
|
||||
LaunchKernel(blocks, kThreads, device)(kernel<kFuseRealSin, kCopyV>, params);
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace qknorm_complex_rope
|
||||
|
||||
} // namespace sglang
|
||||
@@ -147,6 +147,7 @@ tensor copy per residual site.
|
||||
| `fused_rope_rotate_half_bitexact` | Triton | bit-exact (elementwise only) |
|
||||
| `fused_complex_rope` | Triton | preserves CUDA complex64 multiply rounding for contiguous BSHD inputs; Qwen-Image 2.1 verifies its first call against eager |
|
||||
| `rmsnorm_preserve_reduction` | Triton + aten | preserves the FP32 mean reduction and cast-before-weight rounding; fuses only pointwise work for contiguous FP16/BF16 inputs; Qwen-Image 2.1 verifies its first call |
|
||||
| `qknorm_complex_rope_cuda` / `qknorm_complex_rope_pack_` | JIT CUDA | bit-exact vs `RMSNorm(cast_x_before_out_mul=True)` + complex64 RoPE for head_dim 128; the pack variant normalizes Q and K in place behind a prefix, copies prefix K/V (and optionally V) in the same launch, and accepts token-strided views of a packed QKV buffer; Qwen-Image 2.1 verifies its first call |
|
||||
| `fused_interleaved_rope_fp64` | JIT CUDA | bit-exact vs paired SANA-Video fp64 RoPE |
|
||||
| `fused_inplace_helios_qk_rope` | JIT CUDA | bit-exact paired in-place RoPE for Helios' transposed frequency layout |
|
||||
| `ltx2_qknorm_split_rope_cuda` | KDA (JIT CUDA) | close; **validated on B200** |
|
||||
|
||||
@@ -371,6 +371,20 @@ _SPECS: tuple[tuple[str, KernelBackend, str, frozenset, str], ...] = (
|
||||
_CUDA,
|
||||
"Out-of-place fused QK-norm + RoPE (raw q/k preserved).",
|
||||
),
|
||||
(
|
||||
"diffusion.qknorm_complex_rope_cuda",
|
||||
KernelBackend.JIT,
|
||||
"rope.qknorm_complex_rope_jit:qknorm_complex_rope_cuda",
|
||||
_CUDA,
|
||||
"Bit-exact 128-wide RMSNorm + complex RoPE (Qwen-Image 2.1), CUDA.",
|
||||
),
|
||||
(
|
||||
"diffusion.qknorm_complex_rope_pack_",
|
||||
KernelBackend.JIT,
|
||||
"rope.qknorm_complex_rope_jit:qknorm_complex_rope_pack_",
|
||||
_CUDA,
|
||||
"Q/K RMSNorm + complex RoPE with prefix K/V packing into [B, P+S, H, 128] buffers, CUDA.",
|
||||
),
|
||||
(
|
||||
"diffusion.vdn_delta_factors",
|
||||
KernelBackend.JIT,
|
||||
@@ -638,6 +652,10 @@ _EXPORTS: dict[str, str] = {
|
||||
"apply_rotary_embedding": "rope.rotary_triton",
|
||||
"can_use_fused_complex_rope": "rope.complex_rope_triton",
|
||||
"fused_complex_rope": "rope.complex_rope_triton",
|
||||
"can_use_qknorm_complex_rope_cuda": "rope.qknorm_complex_rope_jit",
|
||||
"can_use_qknorm_complex_rope_pack": "rope.qknorm_complex_rope_jit",
|
||||
"qknorm_complex_rope_cuda": "rope.qknorm_complex_rope_jit",
|
||||
"qknorm_complex_rope_pack_": "rope.qknorm_complex_rope_jit",
|
||||
# Tensor layout transformations fused with downstream quantization
|
||||
"try_flux2_token_cat_fp8": "sglang.kernels.kda_kernels.flux2_token_cat_fp8_triton",
|
||||
# Activation-function fusions
|
||||
|
||||
@@ -0,0 +1,230 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Bit-exact CUDA RMSNorm + complex RoPE for Qwen-Image-2.1 attention prep.
|
||||
|
||||
One launch replaces the Triton ``qknorm_complex_rope`` (Q) and
|
||||
``qknorm_complex_rope_kv`` (K, V, prefix) pair: it normalizes and rotates the Q
|
||||
rows in place, writes the normalized/rotated K rows into rows ``[P:P+S]`` of a
|
||||
``[B, P+S, H, 128]`` buffer (in place when the projection already wrote them
|
||||
there), copies the cached prefix K/V into rows ``[0:P]`` and, only when asked,
|
||||
the raw V rows into ``[P:P+S]``.
|
||||
|
||||
Numerical contract (bf16 in and out): fp32 squares summed in aten's vectorized
|
||||
128-wide mean order (lane ``t`` owns elements ``4t..4t+3`` left to right, then
|
||||
a 32-lane shuffle tree), ``rsqrtf(var + eps)``, bf16 rounding after the
|
||||
normalize and after the weight multiply (``RMSNorm(cast_x_before_out_mul=True)``),
|
||||
then explicit-FMA rotation in the orientation ``_fuse_real_sin`` probes for the
|
||||
running GPU. Verified ``torch.equal`` against the eager chain on the production
|
||||
shape ``[1, 4096, 32, 128]`` with an 18-token prefix and on ``[1, 18, 32, 128]``.
|
||||
CUDA only: the ROCm reduction tree differs. Callers verify once through a
|
||||
``BitExactFusionGate`` and keep the eager chain as the fallback.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.jit.utils import cache_once, load_jit, make_cpp_args
|
||||
from sglang.kernels.ops.diffusion.rope.complex_rope_triton import _fuse_real_sin
|
||||
from sglang.srt.utils.custom_op import register_custom_op
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from tvm_ffi.module import Module
|
||||
|
||||
_HEAD_DIM = 128
|
||||
_ROW_ALIGN = 8
|
||||
|
||||
|
||||
@cache_once
|
||||
def _jit_module(fuse_real_sin: bool, copy_v: bool) -> Module:
|
||||
args = make_cpp_args(fuse_real_sin, copy_v)
|
||||
return load_jit(
|
||||
"qknorm_complex_rope",
|
||||
*args,
|
||||
cuda_files=["diffusion/qknorm_complex_rope.cuh"],
|
||||
cuda_wrappers=[
|
||||
("norm_rope", f"qknorm_complex_rope::Kernel<{args}>::run"),
|
||||
("norm_rope_pack", f"qknorm_complex_rope::Kernel<{args}>::run_pack"),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def _is_rows(t: torch.Tensor, device: torch.device, tokens: int | None = None) -> bool:
|
||||
"""``[B, T, H, 128]`` bf16 CUDA tensor whose heads are contiguous; batch/token strides may be padded."""
|
||||
return (
|
||||
isinstance(t, torch.Tensor)
|
||||
and t.dtype is torch.bfloat16
|
||||
and t.is_cuda
|
||||
and t.device == device
|
||||
and t.dim() == 4
|
||||
and t.numel() > 0
|
||||
and t.shape[-1] == _HEAD_DIM
|
||||
and (tokens is None or t.shape[1] == tokens)
|
||||
and t.stride(-1) == 1
|
||||
and t.stride(-2) == _HEAD_DIM
|
||||
and t.stride(1) % (_ROW_ALIGN // 2) == 0
|
||||
and t.stride(0) % (_ROW_ALIGN // 2) == 0
|
||||
and t.data_ptr() % _ROW_ALIGN == 0
|
||||
)
|
||||
|
||||
|
||||
def _is_weight(w: torch.Tensor, device: torch.device) -> bool:
|
||||
return (
|
||||
isinstance(w, torch.Tensor)
|
||||
and w.dtype is torch.bfloat16
|
||||
and w.is_cuda
|
||||
and w.device == device
|
||||
and w.shape == (_HEAD_DIM,)
|
||||
and w.is_contiguous()
|
||||
)
|
||||
|
||||
|
||||
def _is_rope(rope: torch.Tensor, seq: int, device: torch.device) -> bool:
|
||||
return (
|
||||
isinstance(rope, torch.Tensor)
|
||||
and rope.dtype is torch.complex64
|
||||
and rope.device == device
|
||||
and rope.shape == (seq, _HEAD_DIM // 2)
|
||||
and rope.is_contiguous()
|
||||
)
|
||||
|
||||
|
||||
def _cuda_platform(t: torch.Tensor) -> bool:
|
||||
return t.is_cuda and torch.version.hip is None and not torch.compiler.is_compiling()
|
||||
|
||||
|
||||
def _rope_real(rope: torch.Tensor) -> torch.Tensor:
|
||||
return torch.view_as_real(rope).view(rope.shape[0], _HEAD_DIM)
|
||||
|
||||
|
||||
def can_use_qknorm_complex_rope_cuda(
|
||||
x: torch.Tensor, weight: torch.Tensor, rope: torch.Tensor
|
||||
) -> bool:
|
||||
"""Contiguous ``[B, S, H, 128]`` bf16 Q or K with its ``(S, 64)`` complex64 table."""
|
||||
return (
|
||||
_cuda_platform(x)
|
||||
and _is_rows(x, x.device)
|
||||
and x.is_contiguous()
|
||||
and _is_weight(weight, x.device)
|
||||
and _is_rope(rope, x.shape[1], x.device)
|
||||
)
|
||||
|
||||
|
||||
def can_use_qknorm_complex_rope_pack(
|
||||
q: torch.Tensor,
|
||||
k_out: torch.Tensor,
|
||||
v_out: torch.Tensor,
|
||||
q_weight: torch.Tensor,
|
||||
k_weight: torch.Tensor,
|
||||
rope: torch.Tensor,
|
||||
k_prefix: torch.Tensor,
|
||||
v_prefix: torch.Tensor,
|
||||
k_src: Optional[torch.Tensor],
|
||||
v_src: Optional[torch.Tensor],
|
||||
) -> bool:
|
||||
"""Shapes for :func:`qknorm_complex_rope_pack_`; ``None`` sources mean "already in the buffer"."""
|
||||
if not (_cuda_platform(q) and _is_rows(q, q.device)):
|
||||
return False
|
||||
device = q.device
|
||||
batch, seq, heads, _ = q.shape
|
||||
if not (
|
||||
k_prefix.dim() == 4
|
||||
and k_prefix.shape[0] == batch
|
||||
and k_prefix.shape[1] > 0
|
||||
and k_prefix.shape[2] == heads
|
||||
and k_prefix.shape[3] == _HEAD_DIM
|
||||
and k_prefix.is_contiguous()
|
||||
and v_prefix.shape == k_prefix.shape
|
||||
and v_prefix.is_contiguous()
|
||||
and all(
|
||||
t.dtype is torch.bfloat16 and t.device == device
|
||||
for t in (k_prefix, v_prefix)
|
||||
)
|
||||
):
|
||||
return False
|
||||
prefix = k_prefix.shape[1]
|
||||
return (
|
||||
_is_rows(k_out, device, prefix + seq)
|
||||
and _is_rows(v_out, device, prefix + seq)
|
||||
and k_out.shape[0] == batch
|
||||
and v_out.shape[0] == batch
|
||||
and k_out.shape[2] == heads
|
||||
and v_out.shape[2] == heads
|
||||
and (k_src is None or (_is_rows(k_src, device, seq) and k_src.shape == q.shape))
|
||||
and (v_src is None or (_is_rows(v_src, device, seq) and v_src.shape == q.shape))
|
||||
and _is_weight(q_weight, device)
|
||||
and _is_weight(k_weight, device)
|
||||
and _is_rope(rope, seq, device)
|
||||
)
|
||||
|
||||
|
||||
def _fake_norm_rope(x, weight, rope, eps):
|
||||
return torch.empty_like(x)
|
||||
|
||||
|
||||
@register_custom_op(
|
||||
op_name="qknorm_complex_rope_cuda",
|
||||
mutates_args=[],
|
||||
fake_impl=_fake_norm_rope,
|
||||
)
|
||||
def qknorm_complex_rope_cuda(
|
||||
x: torch.Tensor, weight: torch.Tensor, rope: torch.Tensor, eps: float
|
||||
) -> torch.Tensor:
|
||||
"""``rope(rmsnorm(x))`` for one contiguous ``[B, S, H, 128]`` tensor, out of place."""
|
||||
out = torch.empty_like(x)
|
||||
module = _jit_module(_fuse_real_sin(x.device), False)
|
||||
with torch.cuda.device(x.device):
|
||||
module.norm_rope(x, out, weight, _rope_real(rope), float(eps))
|
||||
return out
|
||||
|
||||
|
||||
@register_custom_op(
|
||||
op_name="qknorm_complex_rope_pack_",
|
||||
mutates_args=["q", "k_out", "v_out"],
|
||||
)
|
||||
def qknorm_complex_rope_pack_(
|
||||
q: torch.Tensor,
|
||||
k_out: torch.Tensor,
|
||||
v_out: torch.Tensor,
|
||||
q_weight: torch.Tensor,
|
||||
k_weight: torch.Tensor,
|
||||
rope: torch.Tensor,
|
||||
k_prefix: torch.Tensor,
|
||||
v_prefix: torch.Tensor,
|
||||
k_src: Optional[torch.Tensor],
|
||||
v_src: Optional[torch.Tensor],
|
||||
eps: float,
|
||||
) -> None:
|
||||
"""Normalize/rotate Q in place and K into ``k_out[:, P:]``; fill the prefix rows of ``k_out``/``v_out``.
|
||||
|
||||
``k_src=None`` normalizes ``k_out[:, P:]`` in place (the projection wrote
|
||||
the raw K there). ``v_src=None`` leaves ``v_out[:, P:]`` untouched (the
|
||||
projection wrote V there); otherwise ``v_src`` is copied into it.
|
||||
"""
|
||||
prefix = k_prefix.shape[1]
|
||||
token_rows = k_out[:, prefix:]
|
||||
copy_v = v_src is not None
|
||||
module = _jit_module(_fuse_real_sin(q.device), copy_v)
|
||||
with torch.cuda.device(q.device):
|
||||
module.norm_rope_pack(
|
||||
q,
|
||||
q_weight,
|
||||
token_rows if k_src is None else k_src,
|
||||
k_out,
|
||||
k_weight,
|
||||
v_out,
|
||||
v_src if copy_v else token_rows,
|
||||
_rope_real(rope),
|
||||
k_prefix,
|
||||
v_prefix,
|
||||
float(eps),
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"can_use_qknorm_complex_rope_cuda",
|
||||
"can_use_qknorm_complex_rope_pack",
|
||||
"qknorm_complex_rope_cuda",
|
||||
"qknorm_complex_rope_pack_",
|
||||
]
|
||||
+29
@@ -49,6 +49,8 @@ framework-specific optimization workflow.
|
||||
- `python/sglang/multimodal_gen/runtime/models/vaes/flux2_vae_cuda_opt.py`
|
||||
- `python/sglang/multimodal_gen/runtime/models/vaes/wan_vae_cuda_opt.py`
|
||||
- `python/sglang/multimodal_gen/runtime/models/vaes/autoencoder_kl_qwenimage.py`
|
||||
- `python/sglang/multimodal_gen/runtime/models/dits/qwen_image21.py`
|
||||
- `python/sglang/kernels/ops/diffusion/rope/qknorm_complex_rope_jit.py`
|
||||
- `python/sglang/multimodal_gen/runtime/breakable_cuda_graph/runner.py`
|
||||
- `test/registered/kernels/ops/diffusion/test_modulate.py`
|
||||
- `test/registered/kernels/ops/diffusion/test_norm.py`
|
||||
@@ -286,6 +288,33 @@ framework-specific optimization workflow.
|
||||
contiguity, group layout, and platform before proposing another router
|
||||
kernel.
|
||||
|
||||
17. Qwen-Image 2.1 attention prep
|
||||
- Kernels: `qknorm_complex_rope_cuda`, `qknorm_complex_rope_pack_`.
|
||||
- Locations: `rope/qknorm_complex_rope_jit.py`,
|
||||
`csrc/diffusion/qknorm_complex_rope.cuh`, and
|
||||
`runtime/models/dits/qwen_image21.py`.
|
||||
- Default-on and bit-exact (`BitExactFusionGate`, first call verified against
|
||||
the eager chain): `QwenImage21Attention` projects Q, K and V with one GEMM
|
||||
over a packed `[3C, C]` weight view straight into a `[1, P+S, 3C]` buffer
|
||||
whose rows `[P:]` are the target tokens; one CUDA launch then normalizes Q
|
||||
and K in place (the `RMSNorm(cast_x_before_out_mul=True)` reduction order),
|
||||
applies the complex64 RoPE and copies the prefix K/V into rows `[:P]`, so V
|
||||
is never copied. `apply_qk_norm_rope` uses the same kernel out of place, and
|
||||
the single-element `torch.cat` on the attention output is gone. Unsupported
|
||||
layouts (B > 1, SP > 1, LoRA-wrapped or quantized projections,
|
||||
head_dim != 128) and first-sight mismatches keep the Triton
|
||||
`qknorm_complex_rope` / `qknorm_complex_rope_kv` chain.
|
||||
- Validation: `test/registered/kernels/ops/diffusion/test_qknorm_complex_rope.py`,
|
||||
the `test_qwen21_cuda_qk_rope_*` cases in `test_model_fast_paths.py`, and
|
||||
`python/sglang/multimodal_gen/test/unit/test_qwen_image21_cuda.py`.
|
||||
- Workflow rule: if a Qwen-Image 2.1 trace still shows
|
||||
`_qknorm_complex_rope_onepass_kernel` / `_qknorm_complex_rope_kv_kernel` or
|
||||
three separate Q/K/V GEMMs per block, check the gate logs for a first-sight
|
||||
mismatch and the projection layer type before proposing another kernel. The
|
||||
remaining non-GEMM/attention time per block is the SiLU-mul and the
|
||||
residual-gate + LayerNorm chain, both measured at the DRAM roofline; a fused
|
||||
gated LayerNorm was tried and was slower than the two-kernel chain.
|
||||
|
||||
**Faster CUDA Kernel Usage Points**
|
||||
|
||||
1. sgl-kernel RMSNorm and fused add RMSNorm
|
||||
|
||||
@@ -11,10 +11,14 @@ from sglang.kernels.ops.diffusion import (
|
||||
can_use_fused_complex_rope,
|
||||
can_use_fused_layernorm_modulate,
|
||||
can_use_fused_silu_mul,
|
||||
can_use_qknorm_complex_rope_cuda,
|
||||
can_use_qknorm_complex_rope_pack,
|
||||
can_use_rmsnorm_preserve_reduction,
|
||||
fused_complex_rope,
|
||||
fused_layernorm_modulate,
|
||||
fused_silu_mul_bitexact,
|
||||
qknorm_complex_rope_cuda,
|
||||
qknorm_complex_rope_pack_,
|
||||
residual_gate_add,
|
||||
rmsnorm_preserve_reduction,
|
||||
tensors_equal,
|
||||
@@ -41,6 +45,7 @@ from sglang.multimodal_gen.runtime.layers.attention import LocalAttention, USPAt
|
||||
from sglang.multimodal_gen.runtime.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
RowParallelLinear,
|
||||
UnquantizedLinearMethod,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import (
|
||||
LayerwiseOffloadableModuleMixin,
|
||||
@@ -55,6 +60,18 @@ _ROPE_FUSION = BitExactFusionGate("Qwen-Image 2.1 complex RoPE")
|
||||
_SILU_MUL_FUSION = BitExactFusionGate("Qwen-Image 2.1 SiLU-mul")
|
||||
_QK_ROPE_FUSION = BitExactFusionGate("Qwen-Image 2.1 Q/K RMSNorm + complex RoPE")
|
||||
_KV_ROPE_FUSION = BitExactFusionGate("Qwen-Image 2.1 K RMSNorm + RoPE + KV packing")
|
||||
_QK_ROPE_CUDA_FUSION = BitExactFusionGate(
|
||||
"Qwen-Image 2.1 CUDA Q/K RMSNorm + complex RoPE"
|
||||
)
|
||||
_KV_PACK_CUDA_FUSION = BitExactFusionGate(
|
||||
"Qwen-Image 2.1 CUDA Q/K RMSNorm + RoPE + KV packing"
|
||||
)
|
||||
_KV_PROJECT_INTO_FUSION = BitExactFusionGate(
|
||||
"Qwen-Image 2.1 K/V projection written into the packed KV buffers"
|
||||
)
|
||||
_QKV_PACK_FUSION = BitExactFusionGate(
|
||||
"Qwen-Image 2.1 packed Q/K/V projection written into the packed QKV buffer"
|
||||
)
|
||||
_QK_NORM_FUSION = BitExactFusionGate("Qwen-Image 2.1 Q/K RMSNorm")
|
||||
_MODULATION_FUSION = BitExactFusionGate("Qwen-Image 2.1 LayerNorm modulation")
|
||||
|
||||
@@ -141,6 +158,15 @@ def apply_qk_norm(x, norm):
|
||||
|
||||
def apply_qk_norm_rope(x, norm, rope):
|
||||
fused = None
|
||||
if (
|
||||
can_use_qknorm_complex_rope_cuda(x, norm.weight, rope)
|
||||
and _QK_ROPE_CUDA_FUSION.can_attempt_once()
|
||||
):
|
||||
fused = qknorm_complex_rope_cuda(x, norm.weight, rope, norm.variance_epsilon)
|
||||
if _QK_ROPE_CUDA_FUSION.verified:
|
||||
return fused
|
||||
out = apply_rope(apply_qk_norm(x, norm), rope)
|
||||
return _QK_ROPE_CUDA_FUSION.accept_or_fallback(fused, out, logger=logger)
|
||||
if (
|
||||
can_use_qknorm_complex_rope(x, norm.weight, rope)
|
||||
and _QK_ROPE_FUSION.can_attempt_once()
|
||||
@@ -154,6 +180,41 @@ def apply_qk_norm_rope(x, norm, rope):
|
||||
return out
|
||||
|
||||
|
||||
def can_project_into(layer, x):
|
||||
"""A plain bf16 ColumnParallelLinear whose forward is just F.linear on a resident weight."""
|
||||
return (
|
||||
type(layer) is ColumnParallelLinear
|
||||
and isinstance(layer.quant_method, UnquantizedLinearMethod)
|
||||
and layer.bias is None
|
||||
and not layer.gather_output
|
||||
and not layer._forward_pre_hooks
|
||||
and not layer._forward_hooks
|
||||
and layer.weight.is_cuda
|
||||
and layer.weight.dtype is torch.bfloat16
|
||||
and layer.weight.is_contiguous()
|
||||
and x.dtype is torch.bfloat16
|
||||
and x.is_cuda
|
||||
and x.is_contiguous()
|
||||
and not torch.compiler.is_compiling()
|
||||
)
|
||||
|
||||
|
||||
def project_into(layer, x, out):
|
||||
"""F.linear(x, layer.weight) written into ``out`` (contiguous rows of a larger buffer).
|
||||
|
||||
A contiguous row slice has the same leading dimension as a fresh tensor, so
|
||||
cuBLAS receives the same call as the module forward would issue.
|
||||
"""
|
||||
rows = x.shape[0] * x.shape[1]
|
||||
torch.mm(x.reshape(rows, x.shape[-1]), layer.weight.t(), out=out.view(rows, -1))
|
||||
return out
|
||||
|
||||
|
||||
def cat_outputs(outputs, dim=0):
|
||||
# torch.cat on a one-element list degrades to a full copy of the tensor.
|
||||
return outputs[0] if len(outputs) == 1 else torch.cat(outputs, dim=dim)
|
||||
|
||||
|
||||
def apply_modulation(x, norm, scale):
|
||||
fused = None
|
||||
if (
|
||||
@@ -294,6 +355,74 @@ class QwenImage21Attention(nn.Module):
|
||||
self.heads, self.head_dim, supported_attention_backends=backends
|
||||
)
|
||||
|
||||
def pack_qkv_weights(self):
|
||||
"""Share one [3C, C] storage between to_q/to_k/to_v so one GEMM projects all three.
|
||||
|
||||
The parameters keep their names and shapes (state_dict, LoRA merge and
|
||||
weight updates that copy in place all stay valid); only their storage
|
||||
moves. No reference to the shared storage is kept: ``packed_qkv_weight``
|
||||
re-derives the [3C, C] view from the parameters, so offloading them
|
||||
frees the whole buffer. Layers that are not plain bf16 unquantized
|
||||
projections are left alone and the packed path stays off.
|
||||
"""
|
||||
layers = (self.to_q, self.to_k, self.to_v)
|
||||
if self.packed_qkv_weight() is not None:
|
||||
return
|
||||
if not all(
|
||||
type(layer) is ColumnParallelLinear
|
||||
and isinstance(layer.quant_method, UnquantizedLinearMethod)
|
||||
and layer.bias is None
|
||||
and not layer.gather_output
|
||||
and layer.weight.is_cuda
|
||||
and layer.weight.dtype is torch.bfloat16
|
||||
and layer.weight.is_contiguous()
|
||||
and layer.weight.shape == self.to_q.weight.shape
|
||||
for layer in layers
|
||||
):
|
||||
return
|
||||
packed = torch.cat([layer.weight.data for layer in layers], dim=0)
|
||||
rows = self.to_q.weight.shape[0]
|
||||
for index, layer in enumerate(layers):
|
||||
layer.weight.data = packed[index * rows : (index + 1) * rows]
|
||||
|
||||
def packed_qkv_weight(self):
|
||||
"""The [3C, C] view over to_q/to_k/to_v when they are consecutive slices of one storage, else None."""
|
||||
q, k, v = self.to_q.weight, self.to_k.weight, self.to_v.weight
|
||||
rows, cols = q.shape
|
||||
if not (
|
||||
q.is_cuda
|
||||
and q.dtype is torch.bfloat16
|
||||
and k.dtype is q.dtype
|
||||
and v.dtype is q.dtype
|
||||
and k.shape == q.shape
|
||||
and v.shape == q.shape
|
||||
and q.is_contiguous()
|
||||
and k.is_contiguous()
|
||||
and v.is_contiguous()
|
||||
):
|
||||
return None
|
||||
base = q.untyped_storage().data_ptr()
|
||||
if not (
|
||||
k.untyped_storage().data_ptr() == base
|
||||
and v.untyped_storage().data_ptr() == base
|
||||
):
|
||||
return None
|
||||
stride = rows * cols
|
||||
if not (
|
||||
k.storage_offset() == q.storage_offset() + stride
|
||||
and v.storage_offset() == k.storage_offset() + stride
|
||||
):
|
||||
return None
|
||||
return q.as_strided((3 * rows, cols), (cols, 1))
|
||||
|
||||
def _apply(self, fn, recurse=True):
|
||||
# .to() and CPU offload rebuild every parameter in its own storage;
|
||||
# re-share them once the weights are back on the GPU so the packed
|
||||
# projection survives the round trip (a no-op anywhere else).
|
||||
module = super()._apply(fn, recurse)
|
||||
self.pack_qkv_weights()
|
||||
return module
|
||||
|
||||
def project_qkv(self, x):
|
||||
q = self.to_q(x)[0].unflatten(-1, (self.heads, self.head_dim))
|
||||
k = self.to_k(x)[0].unflatten(-1, (self.heads, self.head_dim))
|
||||
@@ -308,7 +437,150 @@ class QwenImage21Attention(nn.Module):
|
||||
v,
|
||||
)
|
||||
|
||||
def attend_sample(self, q, k, v, rope, prefix, prefix_rope, segments, cache):
|
||||
def _pack_kv_cuda(self, q, k, v, rope, kp, vp, k_out=None, v_out=None):
|
||||
"""One CUDA launch: Q normed+roped in place, K normed+roped and V packed behind the prefix.
|
||||
|
||||
With ``k_out``/``v_out`` the projection already wrote the raw K/V into
|
||||
rows ``[P:]`` (``k``/``v`` are those views), so K is normalized in place
|
||||
and V is not copied. Returns ``(q, k_out, v_out)`` or ``None`` when the
|
||||
layout is unsupported. The first call also runs the eager chain and
|
||||
keeps its result on a mismatch.
|
||||
"""
|
||||
eps = self.norm_k.variance_epsilon
|
||||
if self.norm_q.variance_epsilon != eps:
|
||||
return None
|
||||
prefix = kp.shape[1]
|
||||
if k_out is None:
|
||||
k_out = k.new_empty(k.shape[0], prefix + k.shape[1], *k.shape[2:])
|
||||
v_out = torch.empty_like(k_out)
|
||||
k_src, v_src = k, v
|
||||
else:
|
||||
k_src, v_src = None, None
|
||||
if not can_use_qknorm_complex_rope_pack(
|
||||
q,
|
||||
k_out,
|
||||
v_out,
|
||||
self.norm_q.weight,
|
||||
self.norm_k.weight,
|
||||
rope,
|
||||
kp,
|
||||
vp,
|
||||
k_src,
|
||||
v_src,
|
||||
):
|
||||
return None
|
||||
reference = None
|
||||
if not _KV_PACK_CUDA_FUSION.verified:
|
||||
reference = (
|
||||
apply_rope(apply_qk_norm(q, self.norm_q), rope),
|
||||
torch.cat([kp, apply_rope(apply_qk_norm(k, self.norm_k), rope)], 1),
|
||||
torch.cat([vp, v], 1),
|
||||
)
|
||||
qknorm_complex_rope_pack_(
|
||||
q,
|
||||
k_out,
|
||||
v_out,
|
||||
self.norm_q.weight,
|
||||
self.norm_k.weight,
|
||||
rope,
|
||||
kp,
|
||||
vp,
|
||||
k_src,
|
||||
v_src,
|
||||
eps,
|
||||
)
|
||||
if reference is None:
|
||||
return q, k_out, v_out
|
||||
return _KV_PACK_CUDA_FUSION.accept_or_fallback(
|
||||
(q, k_out, v_out), reference, equal=tensors_equal, logger=logger
|
||||
)
|
||||
|
||||
def _project_qkv_packed(self, x, layouts, caches):
|
||||
"""One GEMM writes Q, K and V into a ``[1, P+S, 3C]`` buffer; ``None`` keeps the plain path.
|
||||
|
||||
Q/K/V become column views with token stride ``3C`` (heads stay contiguous);
|
||||
the fused norm + RoPE kernel and SDPA both accept that layout. The first
|
||||
call compares every slice against the module forwards.
|
||||
"""
|
||||
layers = (self.to_q, self.to_k, self.to_v)
|
||||
packed = self.packed_qkv_weight()
|
||||
if not (
|
||||
packed is not None
|
||||
and x.shape[0] == 1
|
||||
and len(layouts) == 1
|
||||
and get_sp_world_size() == 1
|
||||
and _QKV_PACK_FUSION.can_attempt_once()
|
||||
and all(can_project_into(layer, x) for layer in layers)
|
||||
):
|
||||
return None
|
||||
cache = caches[0]
|
||||
prefix = cache["key"].shape[1] if cache else layouts[0]["prefix_rope"].shape[0]
|
||||
seq = x.shape[1]
|
||||
rows = packed.shape[0] // 3
|
||||
buffer = x.new_empty(1, prefix + seq, 3 * rows)
|
||||
torch.mm(x.view(seq, x.shape[-1]), packed.t(), out=buffer[0, prefix:])
|
||||
q = buffer[:, prefix:, :rows].view(1, seq, self.heads, self.head_dim)
|
||||
k_out = buffer[:, :, rows : 2 * rows].view(
|
||||
1, prefix + seq, self.heads, self.head_dim
|
||||
)
|
||||
v_out = buffer[:, :, 2 * rows :].view(
|
||||
1, prefix + seq, self.heads, self.head_dim
|
||||
)
|
||||
k, v = k_out[:, prefix:], v_out[:, prefix:]
|
||||
if not _QKV_PACK_FUSION.verified:
|
||||
reference = tuple(
|
||||
layer(x)[0].unflatten(-1, (self.heads, self.head_dim))
|
||||
for layer in layers
|
||||
)
|
||||
accepted = _QKV_PACK_FUSION.accept_or_fallback(
|
||||
(q, k, v), reference, equal=tensors_equal, logger=logger
|
||||
)
|
||||
if accepted is reference:
|
||||
return None
|
||||
return q, k, v, k_out, v_out
|
||||
|
||||
def _project_kv_into_buffers(self, x, layouts, caches):
|
||||
"""Project K/V straight into ``[1, P+S, H, D]`` buffers; ``None`` keeps the plain path."""
|
||||
if not (
|
||||
x.shape[0] == 1
|
||||
and len(layouts) == 1
|
||||
and get_sp_world_size() == 1
|
||||
and _KV_PROJECT_INTO_FUSION.can_attempt_once()
|
||||
and can_project_into(self.to_k, x)
|
||||
and can_project_into(self.to_v, x)
|
||||
):
|
||||
return None
|
||||
cache = caches[0]
|
||||
prefix = cache["key"].shape[1] if cache else layouts[0]["prefix_rope"].shape[0]
|
||||
k_out = x.new_empty(1, prefix + x.shape[1], self.heads, self.head_dim)
|
||||
v_out = torch.empty_like(k_out)
|
||||
k = project_into(self.to_k, x, k_out[:, prefix:])
|
||||
v = project_into(self.to_v, x, v_out[:, prefix:])
|
||||
if not _KV_PROJECT_INTO_FUSION.verified:
|
||||
reference = (
|
||||
self.to_k(x)[0].unflatten(-1, (self.heads, self.head_dim)),
|
||||
self.to_v(x)[0].unflatten(-1, (self.heads, self.head_dim)),
|
||||
)
|
||||
accepted = _KV_PROJECT_INTO_FUSION.accept_or_fallback(
|
||||
(k, v), reference, equal=tensors_equal, logger=logger
|
||||
)
|
||||
if accepted is reference:
|
||||
return reference[0], reference[1], None, None
|
||||
return k, v, k_out, v_out
|
||||
|
||||
def attend_sample(
|
||||
self,
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
rope,
|
||||
prefix,
|
||||
prefix_rope,
|
||||
segments,
|
||||
cache,
|
||||
k_out=None,
|
||||
v_out=None,
|
||||
):
|
||||
if cache:
|
||||
kp, vp = cache["key"], cache["value"]
|
||||
prefix_output = None
|
||||
@@ -329,9 +601,13 @@ class QwenImage21Attention(nn.Module):
|
||||
qp[:, start:end], kp[:, :end], vp[:, :end], attn_mask=mask
|
||||
)
|
||||
)
|
||||
prefix_output = self.to_out[0](torch.cat(outputs, dim=1).flatten(2))[0]
|
||||
prefix_output = self.to_out[0](cat_outputs(outputs, dim=1).flatten(2))[0]
|
||||
if cache is not None:
|
||||
cache.update(key=kp, value=vp)
|
||||
if get_sp_world_size() == 1 and _KV_PACK_CUDA_FUSION.can_attempt_once():
|
||||
cuda_packed = self._pack_kv_cuda(q, k, v, rope, kp, vp, k_out, v_out)
|
||||
if cuda_packed is not None:
|
||||
return self.target_attn(*cuda_packed), prefix_output
|
||||
q = apply_qk_norm_rope(q, self.norm_q, rope)
|
||||
packed = None
|
||||
if (
|
||||
@@ -362,7 +638,17 @@ class QwenImage21Attention(nn.Module):
|
||||
|
||||
def forward(self, x, ropes, prefixes, layouts, caches):
|
||||
# batch target projections while retaining each sample's unpadded prefix
|
||||
q, k, v = self.project_qkv(x)
|
||||
packed = self._project_qkv_packed(x, layouts, caches)
|
||||
if packed is not None:
|
||||
q, k, v, k_out, v_out = packed
|
||||
else:
|
||||
buffers = self._project_kv_into_buffers(x, layouts, caches)
|
||||
if buffers is None:
|
||||
q, k, v = self.project_qkv(x)
|
||||
k_out = v_out = None
|
||||
else:
|
||||
q = self.to_q(x)[0].unflatten(-1, (self.heads, self.head_dim))
|
||||
k, v, k_out, v_out = buffers
|
||||
outputs, prefix_outputs = [], []
|
||||
for sample, layout in enumerate(layouts):
|
||||
out, prefix_out = self.attend_sample(
|
||||
@@ -374,10 +660,12 @@ class QwenImage21Attention(nn.Module):
|
||||
layout["prefix_rope"],
|
||||
layout["segments"],
|
||||
caches[sample],
|
||||
k_out=k_out,
|
||||
v_out=v_out,
|
||||
)
|
||||
outputs.append(out)
|
||||
prefix_outputs.append(prefix_out)
|
||||
return self.to_out[0](torch.cat(outputs).flatten(2))[0], prefix_outputs
|
||||
return self.to_out[0](cat_outputs(outputs).flatten(2))[0], prefix_outputs
|
||||
|
||||
|
||||
class QwenImage21TransformerBlock(nn.Module):
|
||||
@@ -498,6 +786,11 @@ class QwenImage21Transformer2DModel(CachableDiT, LayerwiseOffloadableModuleMixin
|
||||
self.norm_out = QwenImage21OutputNorm(ac.hidden_size, ac.eps)
|
||||
self.proj_out = nn.Linear(ac.hidden_size, ac.out_channels, bias=False)
|
||||
|
||||
def post_load_weights(self):
|
||||
super().post_load_weights()
|
||||
for block in self.transformer_blocks:
|
||||
block.attn.pack_qkv_weights()
|
||||
|
||||
def prepare_modulation(self, temb):
|
||||
# All blocks share these gates. Preserve the native tanh and its dtype,
|
||||
# but compute it once per timestep instead of once per block.
|
||||
|
||||
@@ -371,3 +371,183 @@ def test_silu_fusion_mismatch_restores_eager(bf16_model, monkeypatch):
|
||||
torch.testing.assert_close(mlp(x), expected, atol=0, rtol=0)
|
||||
assert gate.disabled and not gate.verified
|
||||
torch.testing.assert_close(mlp(x), expected, atol=0, rtol=0)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def bf16_model_hd128(model):
|
||||
# head_dim 128 is what the CUDA Q/K norm + RoPE + KV packing kernel addresses;
|
||||
# the shared fixture's head_dim 32 never reaches it.
|
||||
config = QwenImage21DitConfig(
|
||||
arch_config=QwenImage21ArchConfig(
|
||||
in_channels=4,
|
||||
out_channels=4,
|
||||
num_layers=2,
|
||||
num_attention_heads=2,
|
||||
attention_head_dim=128,
|
||||
context_in_dim=16,
|
||||
mlp_ratio=2,
|
||||
axes_dims_rope=(16, 56, 56),
|
||||
)
|
||||
)
|
||||
torch.manual_seed(7)
|
||||
result = QwenImage21Transformer2DModel(config, {}).cuda().bfloat16().eval()
|
||||
for name, param in result.named_parameters():
|
||||
torch.nn.init.normal_(param, std=0.02)
|
||||
if name.endswith(("norm_q.weight", "norm_k.weight")):
|
||||
torch.nn.init.normal_(param, mean=1.0, std=0.1)
|
||||
result.post_load_weights()
|
||||
return result
|
||||
|
||||
|
||||
def inputs_hd128(seed, edit):
|
||||
torch.manual_seed(seed)
|
||||
slots = [False] * 3 + ([True, False, False] if edit else [])
|
||||
shapes = ([(1, 2, 4)] if edit else []) + [(1, 4, 4)]
|
||||
kwargs = dict(
|
||||
hidden_states=torch.randn(1, 16, 4, device="cuda"),
|
||||
encoder_hidden_states=torch.randn(1, len(slots), 16, device="cuda"),
|
||||
condition_latents=torch.randn(1, 8, 4, device="cuda") if edit else None,
|
||||
layouts=[build_layout(slots, shapes, (16, 56, 56), "cuda")],
|
||||
prefix_caches=[[{} for _ in range(2)]],
|
||||
timestep=torch.tensor([700.0], device="cuda"),
|
||||
)
|
||||
for key in (
|
||||
"hidden_states",
|
||||
"encoder_hidden_states",
|
||||
"condition_latents",
|
||||
"timestep",
|
||||
):
|
||||
if kwargs[key] is not None:
|
||||
kwargs[key] = kwargs[key].bfloat16()
|
||||
return kwargs
|
||||
|
||||
|
||||
@pytest.mark.parametrize("edit", [False, True])
|
||||
@torch.no_grad()
|
||||
def test_cuda_qk_rope_pack_matches_eager_prefill_and_cached_steps(
|
||||
bf16_model_hd128, edit, monkeypatch
|
||||
):
|
||||
# The CUDA Q/K norm + RoPE + KV packing path must reproduce the Triton/eager
|
||||
# chain bit for bit on the prefill step, on cached steps and under BCG replay.
|
||||
actual_model = bf16_model_hd128
|
||||
kwargs = inputs_hd128(11, edit)
|
||||
reference_kwargs = deepcopy(kwargs)
|
||||
expected = []
|
||||
disabled = BitExactFusionGate("reference")
|
||||
disabled.disable()
|
||||
with monkeypatch.context() as reference, set_forward_context(None, None):
|
||||
reference.setattr(model_module, "_QK_ROPE_CUDA_FUSION", disabled)
|
||||
reference.setattr(model_module, "_KV_PACK_CUDA_FUSION", disabled)
|
||||
reference.setattr(model_module, "_KV_PROJECT_INTO_FUSION", disabled)
|
||||
reference.setattr(model_module, "_QKV_PACK_FUSION", disabled)
|
||||
for timestep in (700, 300, 10):
|
||||
reference_kwargs["timestep"].fill_(timestep)
|
||||
expected.append(actual_model(**reference_kwargs))
|
||||
|
||||
qk_gate = BitExactFusionGate("test CUDA Q/K norm + RoPE")
|
||||
kv_gate = BitExactFusionGate("test CUDA KV packing")
|
||||
project_gate = BitExactFusionGate("test K/V projection into buffers")
|
||||
qkv_gate = BitExactFusionGate("test packed Q/K/V projection")
|
||||
monkeypatch.setattr(model_module, "_QK_ROPE_CUDA_FUSION", qk_gate)
|
||||
monkeypatch.setattr(model_module, "_KV_PACK_CUDA_FUSION", kv_gate)
|
||||
monkeypatch.setattr(model_module, "_KV_PROJECT_INTO_FUSION", project_gate)
|
||||
monkeypatch.setattr(model_module, "_QKV_PACK_FUSION", qkv_gate)
|
||||
with set_forward_context(None, None):
|
||||
for timestep, output in zip((700, 300, 10), expected, strict=True):
|
||||
kwargs["timestep"].fill_(timestep)
|
||||
torch.testing.assert_close(actual_model(**kwargs), output, atol=0, rtol=0)
|
||||
# The CUDA kernels are exact by construction and must have engaged.
|
||||
for gate in (qk_gate, kv_gate):
|
||||
assert gate.verified and not gate.disabled, gate.name
|
||||
# The packed and the direct-write projections are GEMM re-plumbings whose
|
||||
# first-sight compare depends on cuBLAS picking the same kernel for both
|
||||
# shapes; where it does not, the gate declines and the next tier takes over
|
||||
# (the outputs above are bit-exact either way). When the packed projection
|
||||
# verified, it supersedes the per-layer direct write, which never attempts.
|
||||
assert qkv_gate.verified or qkv_gate.disabled
|
||||
if qkv_gate.verified:
|
||||
assert not project_gate.verified and not project_gate.disabled
|
||||
else:
|
||||
assert project_gate.verified or project_gate.disabled
|
||||
for actual, reference in zip(
|
||||
kwargs["prefix_caches"][0], reference_kwargs["prefix_caches"][0], strict=True
|
||||
):
|
||||
for key in ("key", "value"):
|
||||
torch.testing.assert_close(actual[key], reference[key], atol=0, rtol=0)
|
||||
assert (
|
||||
actual[key].untyped_storage().nbytes()
|
||||
== actual[key].numel() * actual[key].element_size()
|
||||
)
|
||||
|
||||
runner = DiffusionBreakableCudaGraphRunner(actual_model, torch.device("cuda"))
|
||||
try:
|
||||
with set_forward_context(None, None):
|
||||
assert runner.capture(**kwargs)
|
||||
kwargs["hidden_states"].add_(0.1)
|
||||
expected = actual_model(**kwargs)
|
||||
torch.testing.assert_close(runner(**kwargs), expected, atol=0, rtol=0)
|
||||
finally:
|
||||
runner.reset()
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def test_cuda_kv_pack_mismatch_restores_reference(bf16_model_hd128, monkeypatch):
|
||||
# A kernel that disagrees with the reference must disable itself and leave
|
||||
# the model output on the Triton/eager chain.
|
||||
kwargs = inputs_hd128(12, False)
|
||||
reference_kwargs = deepcopy(kwargs)
|
||||
disabled = BitExactFusionGate("reference")
|
||||
disabled.disable()
|
||||
with monkeypatch.context() as reference, set_forward_context(None, None):
|
||||
reference.setattr(model_module, "_QK_ROPE_CUDA_FUSION", disabled)
|
||||
reference.setattr(model_module, "_KV_PACK_CUDA_FUSION", disabled)
|
||||
reference.setattr(model_module, "_QKV_PACK_FUSION", disabled)
|
||||
expected = bf16_model_hd128(**reference_kwargs)
|
||||
|
||||
kv_gate = BitExactFusionGate("test mismatched KV packing")
|
||||
monkeypatch.setattr(model_module, "_KV_PACK_CUDA_FUSION", kv_gate)
|
||||
monkeypatch.setattr(model_module, "_QK_ROPE_CUDA_FUSION", disabled)
|
||||
|
||||
def corrupt(q, k_out, v_out, *args):
|
||||
v_out.zero_()
|
||||
|
||||
monkeypatch.setattr(model_module, "qknorm_complex_rope_pack_", corrupt)
|
||||
with set_forward_context(None, None):
|
||||
torch.testing.assert_close(bf16_model_hd128(**kwargs), expected, atol=0, rtol=0)
|
||||
assert kv_gate.disabled and not kv_gate.verified
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def test_packed_qkv_weights_share_storage_and_survive_in_place_updates(
|
||||
bf16_model_hd128,
|
||||
):
|
||||
# pack_qkv_weights must keep parameter names/values and alias the packed buffer,
|
||||
# so a merge-mode LoRA delta written into to_q.weight is what the packed GEMM sees.
|
||||
attn = bf16_model_hd128.transformer_blocks[0].attn
|
||||
packed = attn.packed_qkv_weight()
|
||||
assert packed is not None
|
||||
rows = attn.to_q.weight.shape[0]
|
||||
assert torch.equal(packed[:rows], attn.to_q.weight)
|
||||
assert torch.equal(packed[rows : 2 * rows], attn.to_k.weight)
|
||||
assert torch.equal(packed[2 * rows :], attn.to_v.weight)
|
||||
keys = set(bf16_model_hd128.state_dict().keys())
|
||||
assert "transformer_blocks.0.attn.to_q.weight" in keys
|
||||
assert not any("qkv_weight" in key for key in keys)
|
||||
before = packed[:rows].clone()
|
||||
attn.to_q.weight.add_(1.0)
|
||||
assert torch.equal(attn.packed_qkv_weight()[:rows], before + 1.0)
|
||||
attn.to_q.weight.sub_(1.0)
|
||||
# An offload round trip must not pin the shared CUDA storage: nothing on the
|
||||
# module keeps a device tensor once the parameters moved, and the shared
|
||||
# views come back with the weights.
|
||||
attn.to("cpu")
|
||||
assert attn.packed_qkv_weight() is None
|
||||
assert not any(
|
||||
isinstance(value, torch.Tensor) and value.is_cuda
|
||||
for value in vars(attn).values()
|
||||
)
|
||||
attn.to("cuda")
|
||||
packed = attn.packed_qkv_weight()
|
||||
assert packed is not None
|
||||
assert torch.equal(packed[:rows], attn.to_q.weight)
|
||||
assert torch.equal(packed[2 * rows :], attn.to_v.weight)
|
||||
|
||||
Reference in New Issue
Block a user