diff --git a/python/sglang/kernels/jit/csrc/diffusion/qknorm_complex_rope.cuh b/python/sglang/kernels/jit/csrc/diffusion/qknorm_complex_rope.cuh new file mode 100644 index 0000000000..bd8f0cee0f --- /dev/null +++ b/python/sglang/kernels/jit/csrc/diffusion/qknorm_complex_rope.cuh @@ -0,0 +1,344 @@ +// Bit-exact 128-wide RMSNorm + complex RoPE for Qwen-Image-2.1 attention prep. +// +// Eager chain reproduced for bf16 activations: +// RMSNorm(cast_x_before_out_mul=True): var = mean(float(x)^2) in aten's +// vectorized 128-wide fp32 mean order (lane t owns elements 4t..4t+3, sums +// them left to right, then a 32-lane shuffle tree with offsets 16,8,4,2,1), +// inv = rsqrtf(var + eps), n = bf16(x * inv), o = bf16(n * w); +// complex RoPE on interleaved pairs: out = view_as_real(complex(o) * rope), +// with the FMA orientation PyTorch's complex64 multiply uses on this GPU +// (kFuseRealSin, probed on the Python side). +// One launch handles the Q rows and the K rows (source -> destination, in place +// when they alias), copies the cached prefix K/V into rows [0:P] of the +// [B, P+S, H, 128] K/V buffers and, with kCopyV, the raw V rows into [P:P+S]. + +#pragma once + +#include +#include + +#include +#include +#include + +#include +#include + +namespace sglang { + +namespace qknorm_complex_rope { + +constexpr int kHeadDim = 128; +constexpr int kLaneElems = kHeadDim / device::kWarpThreads; // 4 bf16 per lane +constexpr uint32_t kThreads = 256; +constexpr uint32_t kWarpsPerBlock = kThreads / device::kWarpThreads; +constexpr uint32_t kMaxBlocks = 65535; +constexpr int64_t kVecAlignBytes = 8; + +static_assert(kLaneElems == 4, "the aten mean replica assumes four elements per lane"); + +using RowVec = device::AlignedVector; // 8 bytes +using RopeVec = device::AlignedVector; // cos0 sin0 cos1 sin1 + +/// Strided [B, T, H, 128] tensor: element (b, t, h, d) lives at ptr + b*batch_stride + t*token_stride + h*128 + d. +struct Rows { + const bf16_t* ptr; + int64_t batch_stride; + int64_t token_stride; +}; + +struct MutableRows { + bf16_t* ptr; + int64_t batch_stride; + int64_t token_stride; +}; + +struct Params { + Rows q_src; + MutableRows q_dst; + Rows k_src; + MutableRows k_dst; // [B, P+S, H, 128]; token rows land at P + s + MutableRows v_dst; // [B, P+S, H, 128] + Rows v_src; // read only with kCopyV + const bf16_t* q_weight; + const bf16_t* k_weight; + const bf16_t* k_prefix; // contiguous [B, P, H, 128] + const bf16_t* v_prefix; // contiguous [B, P, H, 128] + const fp32_t* rope; // [S, 128] = view_as_real of the complex64 [S, 64] table + uint32_t batch; + uint32_t seq; + uint32_t heads; + uint32_t prefix; + uint32_t k_rows_enabled; // 0 for the single-tensor entry point + float eps; +}; + +SGL_DEVICE const bf16_t* row_ptr(const Rows& rows, int64_t b, int64_t t, int64_t h) { + return rows.ptr + b * rows.batch_stride + t * rows.token_stride + h * kHeadDim; +} + +SGL_DEVICE bf16_t* row_ptr(const MutableRows& rows, int64_t b, int64_t t, int64_t h) { + return rows.ptr + b * rows.batch_stride + t * rows.token_stride + h * kHeadDim; +} + +/// \brief Normalize and rotate one 128-wide row; every lane owns 4 consecutive elements. +template +SGL_DEVICE void norm_rope_row( + const bf16_t* __restrict__ src, + bf16_t* __restrict__ dst, + const bf16_t* __restrict__ weight, + const fp32_t* __restrict__ rope_row, + float eps, + int lane) { + RowVec x; + RowVec w; + RopeVec r; + x.load(src + lane * kLaneElems); + w.load(weight + lane * kLaneElems); + r.load(rope_row + lane * kLaneElems); + + float value[kLaneElems]; +#pragma unroll + for (int i = 0; i < kLaneElems; ++i) { + value[i] = device::cast(x[i]); + } + // aten vectorized mean: four squares combined left to right ... + float part = __fmul_rn(value[0], value[0]); + part = __fadd_rn(part, __fmul_rn(value[1], value[1])); + part = __fadd_rn(part, __fmul_rn(value[2], value[2])); + part = __fadd_rn(part, __fmul_rn(value[3], value[3])); + // ... then the 32-lane tree with decreasing offsets (matches shfl_down 16..1 for lane 0). +#pragma unroll + for (int offset = device::kWarpThreads / 2; offset > 0; offset >>= 1) { + part = __fadd_rn(part, __shfl_xor_sync(0xffffffffu, part, offset)); + } + const float variance = __fmul_rn(part, 1.0f / kHeadDim); + const float inv = rsqrtf(__fadd_rn(variance, eps)); + + // cast_x_before_out_mul: round after the normalize, then again after the weight. + float out_f32[kLaneElems]; +#pragma unroll + for (int i = 0; i < kLaneElems; ++i) { + const float normalized = device::cast(device::cast(__fmul_rn(value[i], inv))); + out_f32[i] = device::cast(device::cast(__fmul_rn(normalized, device::cast(w[i])))); + } + + RowVec out; +#pragma unroll + for (int p = 0; p < kLaneElems / 2; ++p) { + const float re = out_f32[2 * p]; + const float im = out_f32[2 * p + 1]; + const float cos = r[2 * p]; + const float sin = r[2 * p + 1]; + const float out_re = __fmaf_rn(re, cos, -__fmul_rn(im, sin)); + float out_im; + if constexpr (kFuseRealSin) { + out_im = __fmaf_rn(re, sin, __fmul_rn(im, cos)); + } else { + out_im = __fmaf_rn(im, cos, __fmul_rn(re, sin)); + } + out[2 * p] = device::cast(out_re); + out[2 * p + 1] = device::cast(out_im); + } + out.store(dst + lane * kLaneElems); +} + +SGL_DEVICE void copy_row(const bf16_t* __restrict__ src, bf16_t* __restrict__ dst, int lane) { + RowVec v; + v.load(src + lane * kLaneElems); + v.store(dst + lane * kLaneElems); +} + +/// Work list: Q rows, K rows, prefix-K copies, prefix-V copies, then (kCopyV) V copies. One warp per row. +template +__global__ void kernel(const Params __grid_constant__ params) { + const int lane = threadIdx.x & (device::kWarpThreads - 1); + const uint32_t warp = threadIdx.x >> 5; + const int64_t heads = params.heads; + const int64_t seq = params.seq; + const int64_t prefix = params.prefix; + const int64_t rows_q = static_cast(params.batch) * seq * heads; + const int64_t rows_k = params.k_rows_enabled ? rows_q : 0; + const int64_t rows_p = params.k_rows_enabled ? static_cast(params.batch) * prefix * heads : 0; + const int64_t rows_v = kCopyV ? rows_q : 0; + const int64_t total = rows_q + rows_k + 2 * rows_p + rows_v; + const int64_t stride = static_cast(gridDim.x) * kWarpsPerBlock; + + for (int64_t row = static_cast(blockIdx.x) * kWarpsPerBlock + warp; row < total; row += stride) { + int64_t r = row; + if (r < rows_q) { + const int64_t h = r % heads, t = r / heads, s = t % seq, b = t / seq; + norm_rope_row( + row_ptr(params.q_src, b, s, h), + row_ptr(params.q_dst, b, s, h), + params.q_weight, + params.rope + s * kHeadDim, + params.eps, + lane); + continue; + } + r -= rows_q; + if (r < rows_k) { + const int64_t h = r % heads, t = r / heads, s = t % seq, b = t / seq; + norm_rope_row( + row_ptr(params.k_src, b, s, h), + row_ptr(params.k_dst, b, prefix + s, h), + params.k_weight, + params.rope + s * kHeadDim, + params.eps, + lane); + continue; + } + r -= rows_k; + if (r < rows_p) { + const int64_t h = r % heads, t = r / heads, p = t % prefix, b = t / prefix; + copy_row(params.k_prefix + ((b * prefix + p) * heads + h) * kHeadDim, row_ptr(params.k_dst, b, p, h), lane); + continue; + } + r -= rows_p; + if (r < rows_p) { + const int64_t h = r % heads, t = r / heads, p = t % prefix, b = t / prefix; + copy_row(params.v_prefix + ((b * prefix + p) * heads + h) * kHeadDim, row_ptr(params.v_dst, b, p, h), lane); + continue; + } + if constexpr (kCopyV) { + r -= rows_p; + const int64_t h = r % heads, t = r / heads, s = t % seq, b = t / seq; + copy_row(row_ptr(params.v_src, b, s, h), row_ptr(params.v_dst, b, prefix + s, h), lane); + } + } +} + +template +struct Kernel { + /// \brief Single tensor, out of place: `out = rope(rmsnorm(x))` for a contiguous [B, S, H, 128] bf16 tensor. + static void + run(tvm::ffi::TensorView x, + tvm::ffi::TensorView out, + tvm::ffi::TensorView weight, + tvm::ffi::TensorView rope, + double eps) { + using namespace host; + auto B = SymbolicSize{"batch"}; + auto S = SymbolicSize{"seq"}; + auto H = SymbolicSize{"heads"}; + auto device = SymbolicDevice{}; + device.set_options(); + TensorMatcher({B, S, H, kHeadDim}) + .with_dtype() + .with_device(device) + .ensure_alignment(kVecAlignBytes) + .verify(x) + .verify(out); + TensorMatcher({kHeadDim}).with_dtype().with_device(device).verify(weight); + TensorMatcher({S, kHeadDim}).with_dtype().with_device(device).ensure_alignment(16).verify(rope); + const int64_t batch = B.unwrap(), seq = S.unwrap(), heads = H.unwrap(); + if (batch == 0 || seq == 0 || heads == 0) return; + const int64_t token_stride = heads * kHeadDim; + Params params{}; + params.q_src = Rows{static_cast(x.data_ptr()), seq * token_stride, token_stride}; + params.q_dst = MutableRows{static_cast(out.data_ptr()), seq * token_stride, token_stride}; + params.q_weight = static_cast(weight.data_ptr()); + params.k_weight = params.q_weight; + params.rope = static_cast(rope.data_ptr()); + params.batch = static_cast(batch); + params.seq = static_cast(seq); + params.heads = static_cast(heads); + params.prefix = 0; + params.k_rows_enabled = 0; + params.eps = static_cast(eps); + launch(params, batch * seq * heads, device.unwrap()); + } + + /// \brief Q in place, K source -> rows [P:] of k_out, prefix K/V -> rows [0:P] of k_out/v_out, + /// raw V -> rows [P:] of v_out when kCopyV (v_src is ignored otherwise). + static void run_pack( + tvm::ffi::TensorView q, + tvm::ffi::TensorView q_weight, + tvm::ffi::TensorView k_src, + tvm::ffi::TensorView k_out, + tvm::ffi::TensorView k_weight, + tvm::ffi::TensorView v_out, + tvm::ffi::TensorView v_src, + tvm::ffi::TensorView rope, + tvm::ffi::TensorView k_prefix, + tvm::ffi::TensorView v_prefix, + double eps) { + using namespace host; + auto B = SymbolicSize{"batch"}; + auto S = SymbolicSize{"seq"}; + auto H = SymbolicSize{"heads"}; + auto P = SymbolicSize{"prefix"}; + auto PS = SymbolicSize{"prefix_plus_seq"}; + auto Bq = SymbolicSize{"q_batch_stride"}, Tq = SymbolicSize{"q_token_stride"}; + auto Bk = SymbolicSize{"k_src_batch_stride"}, Tk = SymbolicSize{"k_src_token_stride"}; + auto Bko = SymbolicSize{"k_out_batch_stride"}, Tko = SymbolicSize{"k_out_token_stride"}; + auto Bvo = SymbolicSize{"v_out_batch_stride"}, Tvo = SymbolicSize{"v_out_token_stride"}; + auto Bv = SymbolicSize{"v_src_batch_stride"}, Tv = SymbolicSize{"v_src_token_stride"}; + auto device = SymbolicDevice{}; + device.set_options(); + // TensorMatcher is not copyable, so verify inside the lambda; the member + // template call on a dependent expression needs `template`. + const auto verify_rows = [&](const tvm::ffi::TensorView& t, auto& rows, auto& batch_stride, auto& token_stride) { + TensorMatcher({B, rows, H, kHeadDim}) + .with_strides({batch_stride, token_stride, kHeadDim, 1}) + .template with_dtype() + .with_device(device) + .ensure_alignment(kVecAlignBytes) + .verify(t); + }; + verify_rows(q, S, Bq, Tq); + verify_rows(k_src, S, Bk, Tk); + verify_rows(k_out, PS, Bko, Tko); + verify_rows(v_out, PS, Bvo, Tvo); + if constexpr (kCopyV) { + verify_rows(v_src, S, Bv, Tv); + } + TensorMatcher({B, P, H, kHeadDim}) + .with_dtype() + .with_device(device) + .ensure_alignment(kVecAlignBytes) + .verify(k_prefix) + .verify(v_prefix); + TensorMatcher({kHeadDim}).with_dtype().with_device(device).verify(q_weight).verify(k_weight); + TensorMatcher({S, kHeadDim}).with_dtype().with_device(device).ensure_alignment(16).verify(rope); + const int64_t batch = B.unwrap(), seq = S.unwrap(), heads = H.unwrap(), prefix = P.unwrap(); + CHECK_HOST(PS.unwrap() == prefix + seq) << "k_out/v_out must hold prefix + seq tokens"; + if (batch == 0 || seq == 0 || heads == 0) return; + + Params params{}; + params.q_src = Rows{static_cast(q.data_ptr()), Bq.unwrap(), Tq.unwrap()}; + params.q_dst = MutableRows{static_cast(q.data_ptr()), Bq.unwrap(), Tq.unwrap()}; + params.k_src = Rows{static_cast(k_src.data_ptr()), Bk.unwrap(), Tk.unwrap()}; + params.k_dst = MutableRows{static_cast(k_out.data_ptr()), Bko.unwrap(), Tko.unwrap()}; + params.v_dst = MutableRows{static_cast(v_out.data_ptr()), Bvo.unwrap(), Tvo.unwrap()}; + if constexpr (kCopyV) { + params.v_src = Rows{static_cast(v_src.data_ptr()), Bv.unwrap(), Tv.unwrap()}; + } + params.q_weight = static_cast(q_weight.data_ptr()); + params.k_weight = static_cast(k_weight.data_ptr()); + params.k_prefix = static_cast(k_prefix.data_ptr()); + params.v_prefix = static_cast(v_prefix.data_ptr()); + params.rope = static_cast(rope.data_ptr()); + params.batch = static_cast(batch); + params.seq = static_cast(seq); + params.heads = static_cast(heads); + params.prefix = static_cast(prefix); + params.k_rows_enabled = 1; + params.eps = static_cast(eps); + const int64_t rows = batch * heads * (2 * seq + 2 * prefix + (kCopyV ? seq : 0)); + launch(params, rows, device.unwrap()); + } + + private: + static void launch(const Params& params, int64_t rows, DLDevice device) { + using namespace host; + const int64_t needed = div_ceil(rows, static_cast(kWarpsPerBlock)); + const auto blocks = static_cast(std::min(needed, kMaxBlocks)); + LaunchKernel(blocks, kThreads, device)(kernel, params); + } +}; + +} // namespace qknorm_complex_rope + +} // namespace sglang diff --git a/python/sglang/kernels/ops/diffusion/README.md b/python/sglang/kernels/ops/diffusion/README.md index 212cfd5b93..29d38e25c8 100644 --- a/python/sglang/kernels/ops/diffusion/README.md +++ b/python/sglang/kernels/ops/diffusion/README.md @@ -147,6 +147,7 @@ tensor copy per residual site. | `fused_rope_rotate_half_bitexact` | Triton | bit-exact (elementwise only) | | `fused_complex_rope` | Triton | preserves CUDA complex64 multiply rounding for contiguous BSHD inputs; Qwen-Image 2.1 verifies its first call against eager | | `rmsnorm_preserve_reduction` | Triton + aten | preserves the FP32 mean reduction and cast-before-weight rounding; fuses only pointwise work for contiguous FP16/BF16 inputs; Qwen-Image 2.1 verifies its first call | +| `qknorm_complex_rope_cuda` / `qknorm_complex_rope_pack_` | JIT CUDA | bit-exact vs `RMSNorm(cast_x_before_out_mul=True)` + complex64 RoPE for head_dim 128; the pack variant normalizes Q and K in place behind a prefix, copies prefix K/V (and optionally V) in the same launch, and accepts token-strided views of a packed QKV buffer; Qwen-Image 2.1 verifies its first call | | `fused_interleaved_rope_fp64` | JIT CUDA | bit-exact vs paired SANA-Video fp64 RoPE | | `fused_inplace_helios_qk_rope` | JIT CUDA | bit-exact paired in-place RoPE for Helios' transposed frequency layout | | `ltx2_qknorm_split_rope_cuda` | KDA (JIT CUDA) | close; **validated on B200** | diff --git a/python/sglang/kernels/ops/diffusion/__init__.py b/python/sglang/kernels/ops/diffusion/__init__.py index 351b9c426d..9b918fabf5 100644 --- a/python/sglang/kernels/ops/diffusion/__init__.py +++ b/python/sglang/kernels/ops/diffusion/__init__.py @@ -371,6 +371,20 @@ _SPECS: tuple[tuple[str, KernelBackend, str, frozenset, str], ...] = ( _CUDA, "Out-of-place fused QK-norm + RoPE (raw q/k preserved).", ), + ( + "diffusion.qknorm_complex_rope_cuda", + KernelBackend.JIT, + "rope.qknorm_complex_rope_jit:qknorm_complex_rope_cuda", + _CUDA, + "Bit-exact 128-wide RMSNorm + complex RoPE (Qwen-Image 2.1), CUDA.", + ), + ( + "diffusion.qknorm_complex_rope_pack_", + KernelBackend.JIT, + "rope.qknorm_complex_rope_jit:qknorm_complex_rope_pack_", + _CUDA, + "Q/K RMSNorm + complex RoPE with prefix K/V packing into [B, P+S, H, 128] buffers, CUDA.", + ), ( "diffusion.vdn_delta_factors", KernelBackend.JIT, @@ -638,6 +652,10 @@ _EXPORTS: dict[str, str] = { "apply_rotary_embedding": "rope.rotary_triton", "can_use_fused_complex_rope": "rope.complex_rope_triton", "fused_complex_rope": "rope.complex_rope_triton", + "can_use_qknorm_complex_rope_cuda": "rope.qknorm_complex_rope_jit", + "can_use_qknorm_complex_rope_pack": "rope.qknorm_complex_rope_jit", + "qknorm_complex_rope_cuda": "rope.qknorm_complex_rope_jit", + "qknorm_complex_rope_pack_": "rope.qknorm_complex_rope_jit", # Tensor layout transformations fused with downstream quantization "try_flux2_token_cat_fp8": "sglang.kernels.kda_kernels.flux2_token_cat_fp8_triton", # Activation-function fusions diff --git a/python/sglang/kernels/ops/diffusion/rope/qknorm_complex_rope_jit.py b/python/sglang/kernels/ops/diffusion/rope/qknorm_complex_rope_jit.py new file mode 100644 index 0000000000..f30d0932f6 --- /dev/null +++ b/python/sglang/kernels/ops/diffusion/rope/qknorm_complex_rope_jit.py @@ -0,0 +1,230 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Bit-exact CUDA RMSNorm + complex RoPE for Qwen-Image-2.1 attention prep. + +One launch replaces the Triton ``qknorm_complex_rope`` (Q) and +``qknorm_complex_rope_kv`` (K, V, prefix) pair: it normalizes and rotates the Q +rows in place, writes the normalized/rotated K rows into rows ``[P:P+S]`` of a +``[B, P+S, H, 128]`` buffer (in place when the projection already wrote them +there), copies the cached prefix K/V into rows ``[0:P]`` and, only when asked, +the raw V rows into ``[P:P+S]``. + +Numerical contract (bf16 in and out): fp32 squares summed in aten's vectorized +128-wide mean order (lane ``t`` owns elements ``4t..4t+3`` left to right, then +a 32-lane shuffle tree), ``rsqrtf(var + eps)``, bf16 rounding after the +normalize and after the weight multiply (``RMSNorm(cast_x_before_out_mul=True)``), +then explicit-FMA rotation in the orientation ``_fuse_real_sin`` probes for the +running GPU. Verified ``torch.equal`` against the eager chain on the production +shape ``[1, 4096, 32, 128]`` with an 18-token prefix and on ``[1, 18, 32, 128]``. +CUDA only: the ROCm reduction tree differs. Callers verify once through a +``BitExactFusionGate`` and keep the eager chain as the fallback. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Optional + +import torch + +from sglang.kernels.jit.utils import cache_once, load_jit, make_cpp_args +from sglang.kernels.ops.diffusion.rope.complex_rope_triton import _fuse_real_sin +from sglang.srt.utils.custom_op import register_custom_op + +if TYPE_CHECKING: + from tvm_ffi.module import Module + +_HEAD_DIM = 128 +_ROW_ALIGN = 8 + + +@cache_once +def _jit_module(fuse_real_sin: bool, copy_v: bool) -> Module: + args = make_cpp_args(fuse_real_sin, copy_v) + return load_jit( + "qknorm_complex_rope", + *args, + cuda_files=["diffusion/qknorm_complex_rope.cuh"], + cuda_wrappers=[ + ("norm_rope", f"qknorm_complex_rope::Kernel<{args}>::run"), + ("norm_rope_pack", f"qknorm_complex_rope::Kernel<{args}>::run_pack"), + ], + ) + + +def _is_rows(t: torch.Tensor, device: torch.device, tokens: int | None = None) -> bool: + """``[B, T, H, 128]`` bf16 CUDA tensor whose heads are contiguous; batch/token strides may be padded.""" + return ( + isinstance(t, torch.Tensor) + and t.dtype is torch.bfloat16 + and t.is_cuda + and t.device == device + and t.dim() == 4 + and t.numel() > 0 + and t.shape[-1] == _HEAD_DIM + and (tokens is None or t.shape[1] == tokens) + and t.stride(-1) == 1 + and t.stride(-2) == _HEAD_DIM + and t.stride(1) % (_ROW_ALIGN // 2) == 0 + and t.stride(0) % (_ROW_ALIGN // 2) == 0 + and t.data_ptr() % _ROW_ALIGN == 0 + ) + + +def _is_weight(w: torch.Tensor, device: torch.device) -> bool: + return ( + isinstance(w, torch.Tensor) + and w.dtype is torch.bfloat16 + and w.is_cuda + and w.device == device + and w.shape == (_HEAD_DIM,) + and w.is_contiguous() + ) + + +def _is_rope(rope: torch.Tensor, seq: int, device: torch.device) -> bool: + return ( + isinstance(rope, torch.Tensor) + and rope.dtype is torch.complex64 + and rope.device == device + and rope.shape == (seq, _HEAD_DIM // 2) + and rope.is_contiguous() + ) + + +def _cuda_platform(t: torch.Tensor) -> bool: + return t.is_cuda and torch.version.hip is None and not torch.compiler.is_compiling() + + +def _rope_real(rope: torch.Tensor) -> torch.Tensor: + return torch.view_as_real(rope).view(rope.shape[0], _HEAD_DIM) + + +def can_use_qknorm_complex_rope_cuda( + x: torch.Tensor, weight: torch.Tensor, rope: torch.Tensor +) -> bool: + """Contiguous ``[B, S, H, 128]`` bf16 Q or K with its ``(S, 64)`` complex64 table.""" + return ( + _cuda_platform(x) + and _is_rows(x, x.device) + and x.is_contiguous() + and _is_weight(weight, x.device) + and _is_rope(rope, x.shape[1], x.device) + ) + + +def can_use_qknorm_complex_rope_pack( + q: torch.Tensor, + k_out: torch.Tensor, + v_out: torch.Tensor, + q_weight: torch.Tensor, + k_weight: torch.Tensor, + rope: torch.Tensor, + k_prefix: torch.Tensor, + v_prefix: torch.Tensor, + k_src: Optional[torch.Tensor], + v_src: Optional[torch.Tensor], +) -> bool: + """Shapes for :func:`qknorm_complex_rope_pack_`; ``None`` sources mean "already in the buffer".""" + if not (_cuda_platform(q) and _is_rows(q, q.device)): + return False + device = q.device + batch, seq, heads, _ = q.shape + if not ( + k_prefix.dim() == 4 + and k_prefix.shape[0] == batch + and k_prefix.shape[1] > 0 + and k_prefix.shape[2] == heads + and k_prefix.shape[3] == _HEAD_DIM + and k_prefix.is_contiguous() + and v_prefix.shape == k_prefix.shape + and v_prefix.is_contiguous() + and all( + t.dtype is torch.bfloat16 and t.device == device + for t in (k_prefix, v_prefix) + ) + ): + return False + prefix = k_prefix.shape[1] + return ( + _is_rows(k_out, device, prefix + seq) + and _is_rows(v_out, device, prefix + seq) + and k_out.shape[0] == batch + and v_out.shape[0] == batch + and k_out.shape[2] == heads + and v_out.shape[2] == heads + and (k_src is None or (_is_rows(k_src, device, seq) and k_src.shape == q.shape)) + and (v_src is None or (_is_rows(v_src, device, seq) and v_src.shape == q.shape)) + and _is_weight(q_weight, device) + and _is_weight(k_weight, device) + and _is_rope(rope, seq, device) + ) + + +def _fake_norm_rope(x, weight, rope, eps): + return torch.empty_like(x) + + +@register_custom_op( + op_name="qknorm_complex_rope_cuda", + mutates_args=[], + fake_impl=_fake_norm_rope, +) +def qknorm_complex_rope_cuda( + x: torch.Tensor, weight: torch.Tensor, rope: torch.Tensor, eps: float +) -> torch.Tensor: + """``rope(rmsnorm(x))`` for one contiguous ``[B, S, H, 128]`` tensor, out of place.""" + out = torch.empty_like(x) + module = _jit_module(_fuse_real_sin(x.device), False) + with torch.cuda.device(x.device): + module.norm_rope(x, out, weight, _rope_real(rope), float(eps)) + return out + + +@register_custom_op( + op_name="qknorm_complex_rope_pack_", + mutates_args=["q", "k_out", "v_out"], +) +def qknorm_complex_rope_pack_( + q: torch.Tensor, + k_out: torch.Tensor, + v_out: torch.Tensor, + q_weight: torch.Tensor, + k_weight: torch.Tensor, + rope: torch.Tensor, + k_prefix: torch.Tensor, + v_prefix: torch.Tensor, + k_src: Optional[torch.Tensor], + v_src: Optional[torch.Tensor], + eps: float, +) -> None: + """Normalize/rotate Q in place and K into ``k_out[:, P:]``; fill the prefix rows of ``k_out``/``v_out``. + + ``k_src=None`` normalizes ``k_out[:, P:]`` in place (the projection wrote + the raw K there). ``v_src=None`` leaves ``v_out[:, P:]`` untouched (the + projection wrote V there); otherwise ``v_src`` is copied into it. + """ + prefix = k_prefix.shape[1] + token_rows = k_out[:, prefix:] + copy_v = v_src is not None + module = _jit_module(_fuse_real_sin(q.device), copy_v) + with torch.cuda.device(q.device): + module.norm_rope_pack( + q, + q_weight, + token_rows if k_src is None else k_src, + k_out, + k_weight, + v_out, + v_src if copy_v else token_rows, + _rope_real(rope), + k_prefix, + v_prefix, + float(eps), + ) + + +__all__ = [ + "can_use_qknorm_complex_rope_cuda", + "can_use_qknorm_complex_rope_pack", + "qknorm_complex_rope_cuda", + "qknorm_complex_rope_pack_", +] diff --git a/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-benchmark-profile/existing-fast-paths.md b/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-benchmark-profile/existing-fast-paths.md index 2382df6fd7..680f3c5d3c 100644 --- a/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-benchmark-profile/existing-fast-paths.md +++ b/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-benchmark-profile/existing-fast-paths.md @@ -49,6 +49,8 @@ framework-specific optimization workflow. - `python/sglang/multimodal_gen/runtime/models/vaes/flux2_vae_cuda_opt.py` - `python/sglang/multimodal_gen/runtime/models/vaes/wan_vae_cuda_opt.py` - `python/sglang/multimodal_gen/runtime/models/vaes/autoencoder_kl_qwenimage.py` +- `python/sglang/multimodal_gen/runtime/models/dits/qwen_image21.py` +- `python/sglang/kernels/ops/diffusion/rope/qknorm_complex_rope_jit.py` - `python/sglang/multimodal_gen/runtime/breakable_cuda_graph/runner.py` - `test/registered/kernels/ops/diffusion/test_modulate.py` - `test/registered/kernels/ops/diffusion/test_norm.py` @@ -286,6 +288,33 @@ framework-specific optimization workflow. contiguity, group layout, and platform before proposing another router kernel. +17. Qwen-Image 2.1 attention prep +- Kernels: `qknorm_complex_rope_cuda`, `qknorm_complex_rope_pack_`. +- Locations: `rope/qknorm_complex_rope_jit.py`, + `csrc/diffusion/qknorm_complex_rope.cuh`, and + `runtime/models/dits/qwen_image21.py`. +- Default-on and bit-exact (`BitExactFusionGate`, first call verified against + the eager chain): `QwenImage21Attention` projects Q, K and V with one GEMM + over a packed `[3C, C]` weight view straight into a `[1, P+S, 3C]` buffer + whose rows `[P:]` are the target tokens; one CUDA launch then normalizes Q + and K in place (the `RMSNorm(cast_x_before_out_mul=True)` reduction order), + applies the complex64 RoPE and copies the prefix K/V into rows `[:P]`, so V + is never copied. `apply_qk_norm_rope` uses the same kernel out of place, and + the single-element `torch.cat` on the attention output is gone. Unsupported + layouts (B > 1, SP > 1, LoRA-wrapped or quantized projections, + head_dim != 128) and first-sight mismatches keep the Triton + `qknorm_complex_rope` / `qknorm_complex_rope_kv` chain. +- Validation: `test/registered/kernels/ops/diffusion/test_qknorm_complex_rope.py`, + the `test_qwen21_cuda_qk_rope_*` cases in `test_model_fast_paths.py`, and + `python/sglang/multimodal_gen/test/unit/test_qwen_image21_cuda.py`. +- Workflow rule: if a Qwen-Image 2.1 trace still shows + `_qknorm_complex_rope_onepass_kernel` / `_qknorm_complex_rope_kv_kernel` or + three separate Q/K/V GEMMs per block, check the gate logs for a first-sight + mismatch and the projection layer type before proposing another kernel. The + remaining non-GEMM/attention time per block is the SiLU-mul and the + residual-gate + LayerNorm chain, both measured at the DRAM roofline; a fused + gated LayerNorm was tried and was slower than the two-kernel chain. + **Faster CUDA Kernel Usage Points** 1. sgl-kernel RMSNorm and fused add RMSNorm diff --git a/python/sglang/multimodal_gen/runtime/models/dits/qwen_image21.py b/python/sglang/multimodal_gen/runtime/models/dits/qwen_image21.py index d83fa868a4..228500928f 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/qwen_image21.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/qwen_image21.py @@ -11,10 +11,14 @@ from sglang.kernels.ops.diffusion import ( can_use_fused_complex_rope, can_use_fused_layernorm_modulate, can_use_fused_silu_mul, + can_use_qknorm_complex_rope_cuda, + can_use_qknorm_complex_rope_pack, can_use_rmsnorm_preserve_reduction, fused_complex_rope, fused_layernorm_modulate, fused_silu_mul_bitexact, + qknorm_complex_rope_cuda, + qknorm_complex_rope_pack_, residual_gate_add, rmsnorm_preserve_reduction, tensors_equal, @@ -41,6 +45,7 @@ from sglang.multimodal_gen.runtime.layers.attention import LocalAttention, USPAt from sglang.multimodal_gen.runtime.layers.linear import ( ColumnParallelLinear, RowParallelLinear, + UnquantizedLinearMethod, ) from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import ( LayerwiseOffloadableModuleMixin, @@ -55,6 +60,18 @@ _ROPE_FUSION = BitExactFusionGate("Qwen-Image 2.1 complex RoPE") _SILU_MUL_FUSION = BitExactFusionGate("Qwen-Image 2.1 SiLU-mul") _QK_ROPE_FUSION = BitExactFusionGate("Qwen-Image 2.1 Q/K RMSNorm + complex RoPE") _KV_ROPE_FUSION = BitExactFusionGate("Qwen-Image 2.1 K RMSNorm + RoPE + KV packing") +_QK_ROPE_CUDA_FUSION = BitExactFusionGate( + "Qwen-Image 2.1 CUDA Q/K RMSNorm + complex RoPE" +) +_KV_PACK_CUDA_FUSION = BitExactFusionGate( + "Qwen-Image 2.1 CUDA Q/K RMSNorm + RoPE + KV packing" +) +_KV_PROJECT_INTO_FUSION = BitExactFusionGate( + "Qwen-Image 2.1 K/V projection written into the packed KV buffers" +) +_QKV_PACK_FUSION = BitExactFusionGate( + "Qwen-Image 2.1 packed Q/K/V projection written into the packed QKV buffer" +) _QK_NORM_FUSION = BitExactFusionGate("Qwen-Image 2.1 Q/K RMSNorm") _MODULATION_FUSION = BitExactFusionGate("Qwen-Image 2.1 LayerNorm modulation") @@ -141,6 +158,15 @@ def apply_qk_norm(x, norm): def apply_qk_norm_rope(x, norm, rope): fused = None + if ( + can_use_qknorm_complex_rope_cuda(x, norm.weight, rope) + and _QK_ROPE_CUDA_FUSION.can_attempt_once() + ): + fused = qknorm_complex_rope_cuda(x, norm.weight, rope, norm.variance_epsilon) + if _QK_ROPE_CUDA_FUSION.verified: + return fused + out = apply_rope(apply_qk_norm(x, norm), rope) + return _QK_ROPE_CUDA_FUSION.accept_or_fallback(fused, out, logger=logger) if ( can_use_qknorm_complex_rope(x, norm.weight, rope) and _QK_ROPE_FUSION.can_attempt_once() @@ -154,6 +180,41 @@ def apply_qk_norm_rope(x, norm, rope): return out +def can_project_into(layer, x): + """A plain bf16 ColumnParallelLinear whose forward is just F.linear on a resident weight.""" + return ( + type(layer) is ColumnParallelLinear + and isinstance(layer.quant_method, UnquantizedLinearMethod) + and layer.bias is None + and not layer.gather_output + and not layer._forward_pre_hooks + and not layer._forward_hooks + and layer.weight.is_cuda + and layer.weight.dtype is torch.bfloat16 + and layer.weight.is_contiguous() + and x.dtype is torch.bfloat16 + and x.is_cuda + and x.is_contiguous() + and not torch.compiler.is_compiling() + ) + + +def project_into(layer, x, out): + """F.linear(x, layer.weight) written into ``out`` (contiguous rows of a larger buffer). + + A contiguous row slice has the same leading dimension as a fresh tensor, so + cuBLAS receives the same call as the module forward would issue. + """ + rows = x.shape[0] * x.shape[1] + torch.mm(x.reshape(rows, x.shape[-1]), layer.weight.t(), out=out.view(rows, -1)) + return out + + +def cat_outputs(outputs, dim=0): + # torch.cat on a one-element list degrades to a full copy of the tensor. + return outputs[0] if len(outputs) == 1 else torch.cat(outputs, dim=dim) + + def apply_modulation(x, norm, scale): fused = None if ( @@ -294,6 +355,74 @@ class QwenImage21Attention(nn.Module): self.heads, self.head_dim, supported_attention_backends=backends ) + def pack_qkv_weights(self): + """Share one [3C, C] storage between to_q/to_k/to_v so one GEMM projects all three. + + The parameters keep their names and shapes (state_dict, LoRA merge and + weight updates that copy in place all stay valid); only their storage + moves. No reference to the shared storage is kept: ``packed_qkv_weight`` + re-derives the [3C, C] view from the parameters, so offloading them + frees the whole buffer. Layers that are not plain bf16 unquantized + projections are left alone and the packed path stays off. + """ + layers = (self.to_q, self.to_k, self.to_v) + if self.packed_qkv_weight() is not None: + return + if not all( + type(layer) is ColumnParallelLinear + and isinstance(layer.quant_method, UnquantizedLinearMethod) + and layer.bias is None + and not layer.gather_output + and layer.weight.is_cuda + and layer.weight.dtype is torch.bfloat16 + and layer.weight.is_contiguous() + and layer.weight.shape == self.to_q.weight.shape + for layer in layers + ): + return + packed = torch.cat([layer.weight.data for layer in layers], dim=0) + rows = self.to_q.weight.shape[0] + for index, layer in enumerate(layers): + layer.weight.data = packed[index * rows : (index + 1) * rows] + + def packed_qkv_weight(self): + """The [3C, C] view over to_q/to_k/to_v when they are consecutive slices of one storage, else None.""" + q, k, v = self.to_q.weight, self.to_k.weight, self.to_v.weight + rows, cols = q.shape + if not ( + q.is_cuda + and q.dtype is torch.bfloat16 + and k.dtype is q.dtype + and v.dtype is q.dtype + and k.shape == q.shape + and v.shape == q.shape + and q.is_contiguous() + and k.is_contiguous() + and v.is_contiguous() + ): + return None + base = q.untyped_storage().data_ptr() + if not ( + k.untyped_storage().data_ptr() == base + and v.untyped_storage().data_ptr() == base + ): + return None + stride = rows * cols + if not ( + k.storage_offset() == q.storage_offset() + stride + and v.storage_offset() == k.storage_offset() + stride + ): + return None + return q.as_strided((3 * rows, cols), (cols, 1)) + + def _apply(self, fn, recurse=True): + # .to() and CPU offload rebuild every parameter in its own storage; + # re-share them once the weights are back on the GPU so the packed + # projection survives the round trip (a no-op anywhere else). + module = super()._apply(fn, recurse) + self.pack_qkv_weights() + return module + def project_qkv(self, x): q = self.to_q(x)[0].unflatten(-1, (self.heads, self.head_dim)) k = self.to_k(x)[0].unflatten(-1, (self.heads, self.head_dim)) @@ -308,7 +437,150 @@ class QwenImage21Attention(nn.Module): v, ) - def attend_sample(self, q, k, v, rope, prefix, prefix_rope, segments, cache): + def _pack_kv_cuda(self, q, k, v, rope, kp, vp, k_out=None, v_out=None): + """One CUDA launch: Q normed+roped in place, K normed+roped and V packed behind the prefix. + + With ``k_out``/``v_out`` the projection already wrote the raw K/V into + rows ``[P:]`` (``k``/``v`` are those views), so K is normalized in place + and V is not copied. Returns ``(q, k_out, v_out)`` or ``None`` when the + layout is unsupported. The first call also runs the eager chain and + keeps its result on a mismatch. + """ + eps = self.norm_k.variance_epsilon + if self.norm_q.variance_epsilon != eps: + return None + prefix = kp.shape[1] + if k_out is None: + k_out = k.new_empty(k.shape[0], prefix + k.shape[1], *k.shape[2:]) + v_out = torch.empty_like(k_out) + k_src, v_src = k, v + else: + k_src, v_src = None, None + if not can_use_qknorm_complex_rope_pack( + q, + k_out, + v_out, + self.norm_q.weight, + self.norm_k.weight, + rope, + kp, + vp, + k_src, + v_src, + ): + return None + reference = None + if not _KV_PACK_CUDA_FUSION.verified: + reference = ( + apply_rope(apply_qk_norm(q, self.norm_q), rope), + torch.cat([kp, apply_rope(apply_qk_norm(k, self.norm_k), rope)], 1), + torch.cat([vp, v], 1), + ) + qknorm_complex_rope_pack_( + q, + k_out, + v_out, + self.norm_q.weight, + self.norm_k.weight, + rope, + kp, + vp, + k_src, + v_src, + eps, + ) + if reference is None: + return q, k_out, v_out + return _KV_PACK_CUDA_FUSION.accept_or_fallback( + (q, k_out, v_out), reference, equal=tensors_equal, logger=logger + ) + + def _project_qkv_packed(self, x, layouts, caches): + """One GEMM writes Q, K and V into a ``[1, P+S, 3C]`` buffer; ``None`` keeps the plain path. + + Q/K/V become column views with token stride ``3C`` (heads stay contiguous); + the fused norm + RoPE kernel and SDPA both accept that layout. The first + call compares every slice against the module forwards. + """ + layers = (self.to_q, self.to_k, self.to_v) + packed = self.packed_qkv_weight() + if not ( + packed is not None + and x.shape[0] == 1 + and len(layouts) == 1 + and get_sp_world_size() == 1 + and _QKV_PACK_FUSION.can_attempt_once() + and all(can_project_into(layer, x) for layer in layers) + ): + return None + cache = caches[0] + prefix = cache["key"].shape[1] if cache else layouts[0]["prefix_rope"].shape[0] + seq = x.shape[1] + rows = packed.shape[0] // 3 + buffer = x.new_empty(1, prefix + seq, 3 * rows) + torch.mm(x.view(seq, x.shape[-1]), packed.t(), out=buffer[0, prefix:]) + q = buffer[:, prefix:, :rows].view(1, seq, self.heads, self.head_dim) + k_out = buffer[:, :, rows : 2 * rows].view( + 1, prefix + seq, self.heads, self.head_dim + ) + v_out = buffer[:, :, 2 * rows :].view( + 1, prefix + seq, self.heads, self.head_dim + ) + k, v = k_out[:, prefix:], v_out[:, prefix:] + if not _QKV_PACK_FUSION.verified: + reference = tuple( + layer(x)[0].unflatten(-1, (self.heads, self.head_dim)) + for layer in layers + ) + accepted = _QKV_PACK_FUSION.accept_or_fallback( + (q, k, v), reference, equal=tensors_equal, logger=logger + ) + if accepted is reference: + return None + return q, k, v, k_out, v_out + + def _project_kv_into_buffers(self, x, layouts, caches): + """Project K/V straight into ``[1, P+S, H, D]`` buffers; ``None`` keeps the plain path.""" + if not ( + x.shape[0] == 1 + and len(layouts) == 1 + and get_sp_world_size() == 1 + and _KV_PROJECT_INTO_FUSION.can_attempt_once() + and can_project_into(self.to_k, x) + and can_project_into(self.to_v, x) + ): + return None + cache = caches[0] + prefix = cache["key"].shape[1] if cache else layouts[0]["prefix_rope"].shape[0] + k_out = x.new_empty(1, prefix + x.shape[1], self.heads, self.head_dim) + v_out = torch.empty_like(k_out) + k = project_into(self.to_k, x, k_out[:, prefix:]) + v = project_into(self.to_v, x, v_out[:, prefix:]) + if not _KV_PROJECT_INTO_FUSION.verified: + reference = ( + self.to_k(x)[0].unflatten(-1, (self.heads, self.head_dim)), + self.to_v(x)[0].unflatten(-1, (self.heads, self.head_dim)), + ) + accepted = _KV_PROJECT_INTO_FUSION.accept_or_fallback( + (k, v), reference, equal=tensors_equal, logger=logger + ) + if accepted is reference: + return reference[0], reference[1], None, None + return k, v, k_out, v_out + + def attend_sample( + self, + q, + k, + v, + rope, + prefix, + prefix_rope, + segments, + cache, + k_out=None, + v_out=None, + ): if cache: kp, vp = cache["key"], cache["value"] prefix_output = None @@ -329,9 +601,13 @@ class QwenImage21Attention(nn.Module): qp[:, start:end], kp[:, :end], vp[:, :end], attn_mask=mask ) ) - prefix_output = self.to_out[0](torch.cat(outputs, dim=1).flatten(2))[0] + prefix_output = self.to_out[0](cat_outputs(outputs, dim=1).flatten(2))[0] if cache is not None: cache.update(key=kp, value=vp) + if get_sp_world_size() == 1 and _KV_PACK_CUDA_FUSION.can_attempt_once(): + cuda_packed = self._pack_kv_cuda(q, k, v, rope, kp, vp, k_out, v_out) + if cuda_packed is not None: + return self.target_attn(*cuda_packed), prefix_output q = apply_qk_norm_rope(q, self.norm_q, rope) packed = None if ( @@ -362,7 +638,17 @@ class QwenImage21Attention(nn.Module): def forward(self, x, ropes, prefixes, layouts, caches): # batch target projections while retaining each sample's unpadded prefix - q, k, v = self.project_qkv(x) + packed = self._project_qkv_packed(x, layouts, caches) + if packed is not None: + q, k, v, k_out, v_out = packed + else: + buffers = self._project_kv_into_buffers(x, layouts, caches) + if buffers is None: + q, k, v = self.project_qkv(x) + k_out = v_out = None + else: + q = self.to_q(x)[0].unflatten(-1, (self.heads, self.head_dim)) + k, v, k_out, v_out = buffers outputs, prefix_outputs = [], [] for sample, layout in enumerate(layouts): out, prefix_out = self.attend_sample( @@ -374,10 +660,12 @@ class QwenImage21Attention(nn.Module): layout["prefix_rope"], layout["segments"], caches[sample], + k_out=k_out, + v_out=v_out, ) outputs.append(out) prefix_outputs.append(prefix_out) - return self.to_out[0](torch.cat(outputs).flatten(2))[0], prefix_outputs + return self.to_out[0](cat_outputs(outputs).flatten(2))[0], prefix_outputs class QwenImage21TransformerBlock(nn.Module): @@ -498,6 +786,11 @@ class QwenImage21Transformer2DModel(CachableDiT, LayerwiseOffloadableModuleMixin self.norm_out = QwenImage21OutputNorm(ac.hidden_size, ac.eps) self.proj_out = nn.Linear(ac.hidden_size, ac.out_channels, bias=False) + def post_load_weights(self): + super().post_load_weights() + for block in self.transformer_blocks: + block.attn.pack_qkv_weights() + def prepare_modulation(self, temb): # All blocks share these gates. Preserve the native tanh and its dtype, # but compute it once per timestep instead of once per block. diff --git a/python/sglang/multimodal_gen/test/unit/test_qwen_image21_cuda.py b/python/sglang/multimodal_gen/test/unit/test_qwen_image21_cuda.py index 3b672c50fc..b73d0a742b 100644 --- a/python/sglang/multimodal_gen/test/unit/test_qwen_image21_cuda.py +++ b/python/sglang/multimodal_gen/test/unit/test_qwen_image21_cuda.py @@ -371,3 +371,183 @@ def test_silu_fusion_mismatch_restores_eager(bf16_model, monkeypatch): torch.testing.assert_close(mlp(x), expected, atol=0, rtol=0) assert gate.disabled and not gate.verified torch.testing.assert_close(mlp(x), expected, atol=0, rtol=0) + + +@pytest.fixture +def bf16_model_hd128(model): + # head_dim 128 is what the CUDA Q/K norm + RoPE + KV packing kernel addresses; + # the shared fixture's head_dim 32 never reaches it. + config = QwenImage21DitConfig( + arch_config=QwenImage21ArchConfig( + in_channels=4, + out_channels=4, + num_layers=2, + num_attention_heads=2, + attention_head_dim=128, + context_in_dim=16, + mlp_ratio=2, + axes_dims_rope=(16, 56, 56), + ) + ) + torch.manual_seed(7) + result = QwenImage21Transformer2DModel(config, {}).cuda().bfloat16().eval() + for name, param in result.named_parameters(): + torch.nn.init.normal_(param, std=0.02) + if name.endswith(("norm_q.weight", "norm_k.weight")): + torch.nn.init.normal_(param, mean=1.0, std=0.1) + result.post_load_weights() + return result + + +def inputs_hd128(seed, edit): + torch.manual_seed(seed) + slots = [False] * 3 + ([True, False, False] if edit else []) + shapes = ([(1, 2, 4)] if edit else []) + [(1, 4, 4)] + kwargs = dict( + hidden_states=torch.randn(1, 16, 4, device="cuda"), + encoder_hidden_states=torch.randn(1, len(slots), 16, device="cuda"), + condition_latents=torch.randn(1, 8, 4, device="cuda") if edit else None, + layouts=[build_layout(slots, shapes, (16, 56, 56), "cuda")], + prefix_caches=[[{} for _ in range(2)]], + timestep=torch.tensor([700.0], device="cuda"), + ) + for key in ( + "hidden_states", + "encoder_hidden_states", + "condition_latents", + "timestep", + ): + if kwargs[key] is not None: + kwargs[key] = kwargs[key].bfloat16() + return kwargs + + +@pytest.mark.parametrize("edit", [False, True]) +@torch.no_grad() +def test_cuda_qk_rope_pack_matches_eager_prefill_and_cached_steps( + bf16_model_hd128, edit, monkeypatch +): + # The CUDA Q/K norm + RoPE + KV packing path must reproduce the Triton/eager + # chain bit for bit on the prefill step, on cached steps and under BCG replay. + actual_model = bf16_model_hd128 + kwargs = inputs_hd128(11, edit) + reference_kwargs = deepcopy(kwargs) + expected = [] + disabled = BitExactFusionGate("reference") + disabled.disable() + with monkeypatch.context() as reference, set_forward_context(None, None): + reference.setattr(model_module, "_QK_ROPE_CUDA_FUSION", disabled) + reference.setattr(model_module, "_KV_PACK_CUDA_FUSION", disabled) + reference.setattr(model_module, "_KV_PROJECT_INTO_FUSION", disabled) + reference.setattr(model_module, "_QKV_PACK_FUSION", disabled) + for timestep in (700, 300, 10): + reference_kwargs["timestep"].fill_(timestep) + expected.append(actual_model(**reference_kwargs)) + + qk_gate = BitExactFusionGate("test CUDA Q/K norm + RoPE") + kv_gate = BitExactFusionGate("test CUDA KV packing") + project_gate = BitExactFusionGate("test K/V projection into buffers") + qkv_gate = BitExactFusionGate("test packed Q/K/V projection") + monkeypatch.setattr(model_module, "_QK_ROPE_CUDA_FUSION", qk_gate) + monkeypatch.setattr(model_module, "_KV_PACK_CUDA_FUSION", kv_gate) + monkeypatch.setattr(model_module, "_KV_PROJECT_INTO_FUSION", project_gate) + monkeypatch.setattr(model_module, "_QKV_PACK_FUSION", qkv_gate) + with set_forward_context(None, None): + for timestep, output in zip((700, 300, 10), expected, strict=True): + kwargs["timestep"].fill_(timestep) + torch.testing.assert_close(actual_model(**kwargs), output, atol=0, rtol=0) + # The CUDA kernels are exact by construction and must have engaged. + for gate in (qk_gate, kv_gate): + assert gate.verified and not gate.disabled, gate.name + # The packed and the direct-write projections are GEMM re-plumbings whose + # first-sight compare depends on cuBLAS picking the same kernel for both + # shapes; where it does not, the gate declines and the next tier takes over + # (the outputs above are bit-exact either way). When the packed projection + # verified, it supersedes the per-layer direct write, which never attempts. + assert qkv_gate.verified or qkv_gate.disabled + if qkv_gate.verified: + assert not project_gate.verified and not project_gate.disabled + else: + assert project_gate.verified or project_gate.disabled + for actual, reference in zip( + kwargs["prefix_caches"][0], reference_kwargs["prefix_caches"][0], strict=True + ): + for key in ("key", "value"): + torch.testing.assert_close(actual[key], reference[key], atol=0, rtol=0) + assert ( + actual[key].untyped_storage().nbytes() + == actual[key].numel() * actual[key].element_size() + ) + + runner = DiffusionBreakableCudaGraphRunner(actual_model, torch.device("cuda")) + try: + with set_forward_context(None, None): + assert runner.capture(**kwargs) + kwargs["hidden_states"].add_(0.1) + expected = actual_model(**kwargs) + torch.testing.assert_close(runner(**kwargs), expected, atol=0, rtol=0) + finally: + runner.reset() + + +@torch.no_grad() +def test_cuda_kv_pack_mismatch_restores_reference(bf16_model_hd128, monkeypatch): + # A kernel that disagrees with the reference must disable itself and leave + # the model output on the Triton/eager chain. + kwargs = inputs_hd128(12, False) + reference_kwargs = deepcopy(kwargs) + disabled = BitExactFusionGate("reference") + disabled.disable() + with monkeypatch.context() as reference, set_forward_context(None, None): + reference.setattr(model_module, "_QK_ROPE_CUDA_FUSION", disabled) + reference.setattr(model_module, "_KV_PACK_CUDA_FUSION", disabled) + reference.setattr(model_module, "_QKV_PACK_FUSION", disabled) + expected = bf16_model_hd128(**reference_kwargs) + + kv_gate = BitExactFusionGate("test mismatched KV packing") + monkeypatch.setattr(model_module, "_KV_PACK_CUDA_FUSION", kv_gate) + monkeypatch.setattr(model_module, "_QK_ROPE_CUDA_FUSION", disabled) + + def corrupt(q, k_out, v_out, *args): + v_out.zero_() + + monkeypatch.setattr(model_module, "qknorm_complex_rope_pack_", corrupt) + with set_forward_context(None, None): + torch.testing.assert_close(bf16_model_hd128(**kwargs), expected, atol=0, rtol=0) + assert kv_gate.disabled and not kv_gate.verified + + +@torch.no_grad() +def test_packed_qkv_weights_share_storage_and_survive_in_place_updates( + bf16_model_hd128, +): + # pack_qkv_weights must keep parameter names/values and alias the packed buffer, + # so a merge-mode LoRA delta written into to_q.weight is what the packed GEMM sees. + attn = bf16_model_hd128.transformer_blocks[0].attn + packed = attn.packed_qkv_weight() + assert packed is not None + rows = attn.to_q.weight.shape[0] + assert torch.equal(packed[:rows], attn.to_q.weight) + assert torch.equal(packed[rows : 2 * rows], attn.to_k.weight) + assert torch.equal(packed[2 * rows :], attn.to_v.weight) + keys = set(bf16_model_hd128.state_dict().keys()) + assert "transformer_blocks.0.attn.to_q.weight" in keys + assert not any("qkv_weight" in key for key in keys) + before = packed[:rows].clone() + attn.to_q.weight.add_(1.0) + assert torch.equal(attn.packed_qkv_weight()[:rows], before + 1.0) + attn.to_q.weight.sub_(1.0) + # An offload round trip must not pin the shared CUDA storage: nothing on the + # module keeps a device tensor once the parameters moved, and the shared + # views come back with the weights. + attn.to("cpu") + assert attn.packed_qkv_weight() is None + assert not any( + isinstance(value, torch.Tensor) and value.is_cuda + for value in vars(attn).values() + ) + attn.to("cuda") + packed = attn.packed_qkv_weight() + assert packed is not None + assert torch.equal(packed[:rows], attn.to_q.weight) + assert torch.equal(packed[2 * rows :], attn.to_v.weight) diff --git a/test/registered/kernels/ops/diffusion/test_complex_rope.py b/test/registered/kernels/ops/diffusion/test_complex_rope.py index 5ecf2eeb6d..ef6747f21f 100644 --- a/test/registered/kernels/ops/diffusion/test_complex_rope.py +++ b/test/registered/kernels/ops/diffusion/test_complex_rope.py @@ -7,12 +7,17 @@ import torch from sglang.kernels.ops.diffusion import ( BitExactFusionGate, can_use_fused_complex_rope, + can_use_qknorm_complex_rope_cuda, + can_use_qknorm_complex_rope_pack, fused_complex_rope, + qknorm_complex_rope_cuda, + qknorm_complex_rope_pack_, ) from sglang.multimodal_gen.runtime.models.dits import qwen_image21 +from sglang.srt.layers.layernorm import RMSNorm from sglang.test.ci.ci_register import register_cuda_ci -register_cuda_ci(est_time=20, stage="base-b-kernel-unit", runner_config="1-gpu-large") +register_cuda_ci(est_time=60, stage="base-b-kernel-unit", runner_config="1-gpu-large") pytestmark = pytest.mark.skipif( not torch.cuda.is_available() or torch.version.hip is not None, @@ -87,5 +92,282 @@ def test_qwen21_rope_first_sight_verification(monkeypatch): assert gate.disabled and not gate.verified +# ------------------------------------------------------------------------- +# Qwen-Image 2.1 -- CUDA Q/K RMSNorm + complex RoPE, and the packed K/V variant +# ------------------------------------------------------------------------- + + +EPS = 1e-6 + +HEAD_DIM = 128 + + +def make_norm(): + norm = RMSNorm(HEAD_DIM, EPS, cast_x_before_out_mul=True, force_native=True).to( + device="cuda", dtype=torch.bfloat16 + ) + norm.weight.normal_() + return norm + + +def make_rope(seq, scale=20.0): + angles = torch.randn(seq, HEAD_DIM // 2, device="cuda") * scale + return torch.polar(torch.ones_like(angles), angles) + + +def eager_norm_rope(x, norm, rope): + normed = norm(x) + z = torch.view_as_complex(normed.float().reshape(*normed.shape[:-1], -1, 2)) + return torch.view_as_real(z * rope[None, :, None]).flatten(-2).to(x.dtype) + + +def assert_bits_equal(actual, expected): + assert actual.dtype is expected.dtype and actual.shape == expected.shape + assert torch.equal(actual.view(torch.int16), expected.view(torch.int16)) + + +@torch.no_grad() +@pytest.mark.parametrize( + "shape", [(1, 4096, 32, 128), (1, 18, 32, 128), (2, 17, 4, 128), (1, 1, 1, 128)] +) +@pytest.mark.parametrize("amplitude", [1e-3, 1.0, 40.0]) +def test_norm_rope_matches_eager(shape, amplitude): + torch.manual_seed(0) + norm = make_norm() + x = torch.randn(shape, device="cuda", dtype=torch.bfloat16) * amplitude + rope = make_rope(shape[1]) + assert can_use_qknorm_complex_rope_cuda(x, norm.weight, rope) + out = qknorm_complex_rope_cuda(x, norm.weight, rope, EPS) + assert_bits_equal(out, eager_norm_rope(x, norm, rope)) + + +@torch.no_grad() +def test_norm_rope_is_sensitive_to_fma_orientation(): + # The two complex-multiply FMA orientations differ on a large random sample; + # the equality test above therefore also pins the orientation probe. + torch.manual_seed(1) + x = torch.randn(1, 4096, 32, 128, device="cuda", dtype=torch.bfloat16) + rope = make_rope(4096) + normed = x # rotation only: pretend the norm is the identity for this check + z = torch.view_as_complex(normed.float().reshape(*normed.shape[:-1], -1, 2)) + re, im = z.real, z.imag + c, s = rope.real[None, :, None], rope.imag[None, :, None] + imag_a = torch.addcmul(im * c, re, s).to(torch.bfloat16) + imag_b = torch.addcmul(re * s, im, c).to(torch.bfloat16) + assert not torch.equal(imag_a, imag_b) + + +def _pack_inputs(batch, seq, prefix, heads): + torch.manual_seed(2) + q = torch.randn(batch, seq, heads, HEAD_DIM, device="cuda", dtype=torch.bfloat16) + k = torch.randn_like(q) + v = torch.randn_like(q) + kp = torch.randn( + batch, prefix, heads, HEAD_DIM, device="cuda", dtype=torch.bfloat16 + ) + vp = torch.randn_like(kp) + rope = make_rope(seq) + return q, k, v, kp, vp, rope + + +@torch.no_grad() +@pytest.mark.parametrize("batch,seq,prefix,heads", [(1, 4096, 18, 32), (2, 17, 5, 4)]) +@pytest.mark.parametrize("in_place", [False, True]) +def test_pack_matches_eager(batch, seq, prefix, heads, in_place): + q, k, v, kp, vp, rope = _pack_inputs(batch, seq, prefix, heads) + norm_q, norm_k = make_norm(), make_norm() + expected_q = eager_norm_rope(q, norm_q, rope) + expected_k = torch.cat([kp, eager_norm_rope(k, norm_k, rope)], dim=1) + expected_v = torch.cat([vp, v], dim=1) + + k_out = torch.empty( + batch, prefix + seq, heads, HEAD_DIM, device="cuda", dtype=torch.bfloat16 + ) + v_out = torch.empty_like(k_out) + if in_place: + # the projection wrote raw K and V into the token rows already + k_out[:, prefix:].copy_(k) + v_out[:, prefix:].copy_(v) + k_src, v_src = None, None + else: + k_src, v_src = k, v + q_work = q.clone() + assert can_use_qknorm_complex_rope_pack( + q_work, k_out, v_out, norm_q.weight, norm_k.weight, rope, kp, vp, k_src, v_src + ) + qknorm_complex_rope_pack_( + q_work, + k_out, + v_out, + norm_q.weight, + norm_k.weight, + rope, + kp, + vp, + k_src, + v_src, + EPS, + ) + assert_bits_equal(q_work, expected_q) + assert_bits_equal(k_out, expected_k) + assert_bits_equal(v_out, expected_v) + + +@torch.no_grad() +def test_pack_accepts_padded_token_stride(): + # Column views of a wider [B, P+S, 3C] buffer: heads contiguous, token stride 3C. + batch, seq, prefix, heads = 1, 33, 4, 8 + q, k, v, kp, vp, rope = _pack_inputs(batch, seq, prefix, heads) + norm_q, norm_k = make_norm(), make_norm() + wide = torch.empty( + batch, prefix + seq, 3 * heads * HEAD_DIM, device="cuda", dtype=torch.bfloat16 + ) + hd = heads * HEAD_DIM + q_view = wide[:, prefix:, :hd].view(batch, seq, heads, HEAD_DIM) + k_out = wide[:, :, hd : 2 * hd].view(batch, prefix + seq, heads, HEAD_DIM) + v_out = wide[:, :, 2 * hd :].view(batch, prefix + seq, heads, HEAD_DIM) + q_view.copy_(q) + k_out[:, prefix:].copy_(k) + v_out[:, prefix:].copy_(v) + assert can_use_qknorm_complex_rope_pack( + q_view, k_out, v_out, norm_q.weight, norm_k.weight, rope, kp, vp, None, None + ) + qknorm_complex_rope_pack_( + q_view, + k_out, + v_out, + norm_q.weight, + norm_k.weight, + rope, + kp, + vp, + None, + None, + EPS, + ) + assert_bits_equal(q_view, eager_norm_rope(q, norm_q, rope)) + assert_bits_equal(k_out, torch.cat([kp, eager_norm_rope(k, norm_k, rope)], dim=1)) + assert_bits_equal(v_out, torch.cat([vp, v], dim=1)) + + +@torch.no_grad() +def test_predicates_reject_unsupported_layouts(): + norm = make_norm() + x = torch.randn(1, 17, 4, HEAD_DIM, device="cuda", dtype=torch.bfloat16) + rope = make_rope(17) + assert can_use_qknorm_complex_rope_cuda(x, norm.weight, rope) + assert not can_use_qknorm_complex_rope_cuda(x.cpu(), norm.weight.cpu(), rope.cpu()) + assert not can_use_qknorm_complex_rope_cuda(x.half(), norm.weight.half(), rope) + assert not can_use_qknorm_complex_rope_cuda( + x[..., :64], norm.weight[:64], rope[:, :32] + ) + assert not can_use_qknorm_complex_rope_cuda(x.transpose(1, 2), norm.weight, rope) + assert not can_use_qknorm_complex_rope_cuda(x, norm.weight, rope[:-1]) + assert not can_use_qknorm_complex_rope_cuda( + x, norm.weight, rope.to(torch.complex128) + ) + assert not can_use_qknorm_complex_rope_cuda(x, norm.weight.float(), rope) + k_out = torch.empty(1, 17 + 3, 4, HEAD_DIM, device="cuda", dtype=torch.bfloat16) + kp = torch.randn(1, 3, 4, HEAD_DIM, device="cuda", dtype=torch.bfloat16) + assert can_use_qknorm_complex_rope_pack( + x, k_out, k_out.clone(), norm.weight, norm.weight, rope, kp, kp, None, None + ) + # wrong prefix + seq length, batch mismatch, non-contiguous prefix, short K source + assert not can_use_qknorm_complex_rope_pack( + x, + k_out[:, :-1], + k_out.clone(), + norm.weight, + norm.weight, + rope, + kp, + kp, + None, + None, + ) + assert not can_use_qknorm_complex_rope_pack( + x, + k_out, + k_out.clone(), + norm.weight, + norm.weight, + rope, + kp.expand(2, -1, -1, -1), + kp, + None, + None, + ) + assert not can_use_qknorm_complex_rope_pack( + x, + k_out, + k_out.clone(), + norm.weight, + norm.weight, + rope, + kp.transpose(1, 2), + kp, + None, + None, + ) + assert not can_use_qknorm_complex_rope_pack( + x, k_out, k_out.clone(), norm.weight, norm.weight, rope, kp, kp, x[:, :-1], None + ) + + +@torch.no_grad() +def test_pack_graph_capture_and_replay(): + batch, seq, prefix, heads = 1, 65, 3, 8 + q, k, v, kp, vp, rope = _pack_inputs(batch, seq, prefix, heads) + norm_q, norm_k = make_norm(), make_norm() + q_work = q.clone() + k_out = torch.empty( + batch, prefix + seq, heads, HEAD_DIM, device="cuda", dtype=torch.bfloat16 + ) + v_out = torch.empty_like(k_out) + k_out[:, prefix:].copy_(k) + v_out[:, prefix:].copy_(v) + # warm the JIT module outside the capture + qknorm_complex_rope_pack_( + q_work.clone(), + k_out.clone(), + v_out.clone(), + norm_q.weight, + norm_k.weight, + rope, + kp, + vp, + None, + None, + EPS, + ) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + qknorm_complex_rope_pack_( + q_work, + k_out, + v_out, + norm_q.weight, + norm_k.weight, + rope, + kp, + vp, + None, + None, + EPS, + ) + # new inputs through the captured buffers + q2, k2, v2, kp2, vp2, _ = _pack_inputs(batch, seq, prefix, heads) + q_work.copy_(q2) + k_out[:, prefix:].copy_(k2) + v_out[:, prefix:].copy_(v2) + kp.copy_(kp2) + vp.copy_(vp2) + graph.replay() + torch.cuda.synchronize() + assert_bits_equal(q_work, eager_norm_rope(q2, norm_q, rope)) + assert_bits_equal(k_out, torch.cat([kp2, eager_norm_rope(k2, norm_k, rope)], dim=1)) + assert_bits_equal(v_out, torch.cat([vp2, v2], dim=1)) + + if __name__ == "__main__": sys.exit(pytest.main([__file__, "-v"])) diff --git a/test/registered/kernels/ops/diffusion/test_model_fast_paths.py b/test/registered/kernels/ops/diffusion/test_model_fast_paths.py index c7ee3618ec..b2934ee3bb 100644 --- a/test/registered/kernels/ops/diffusion/test_model_fast_paths.py +++ b/test/registered/kernels/ops/diffusion/test_model_fast_paths.py @@ -73,6 +73,7 @@ from sglang.multimodal_gen.runtime.layers.layernorm import ( RMSNormNoWeight, apply_qk_norm, ) +from sglang.multimodal_gen.runtime.layers.linear import MergedColumnParallelLinear from sglang.multimodal_gen.runtime.layers.rotary_embedding.utils import ( _apply_rotary_emb, ) @@ -523,9 +524,12 @@ class TestFlux2EagerFusions(CustomTestCase): # ------------------------------------------------------------------------- -class _PackedAddedQKV(nn.Module): +class _PackedAddedQKV(MergedColumnParallelLinear): + # The lossless split only applies to a MergedColumnParallelLinear (LoRA + # wrappers are called as they are), so the fake must be one; its __init__ + # is skipped because it needs no parallel state. def __init__(self, dim: int): - super().__init__() + nn.Module.__init__(self) self.output_partition_sizes = [dim, dim, dim] self.quant_config = None self.weight = nn.Parameter( @@ -543,18 +547,58 @@ class _PackedAddedQKV(nn.Module): return F.linear(x, self.weight, self.bias), None -def test_qwen_added_qkv_lossless_uses_three_reference_gemms(): - torch.manual_seed(20260831) - dim = 64 - x = torch.randn(1, 17, dim, device="cuda", dtype=torch.bfloat16) - packed = _PackedAddedQKV(dim) +class _WrappedAddedQKV(nn.Module): + # Stands for a LoRA-wrapped projection: not a MergedColumnParallelLinear, + # so the lossless path must call it instead of slicing its weight. + def __init__(self, dim: int): + super().__init__() + self.weight = nn.Parameter( + torch.randn(3 * dim, dim, device="cuda", dtype=torch.bfloat16), + requires_grad=False, + ) + self.bias = nn.Parameter( + torch.randn(3 * dim, device="cuda", dtype=torch.bfloat16), + requires_grad=False, + ) + self.calls = 0 + def forward(self, x): + self.calls += 1 + return F.linear(x, self.weight, self.bias), None + + +def _added_qkv_attention(packed): attention = QwenImageCrossAttention.__new__(QwenImageCrossAttention) nn.Module.__init__(attention) attention.use_fused_added_qkv = True attention._unquantized_added_qkv_is_packed = True attention.to_added_qkv = packed mark_qwen_image_added_qkv_site(attention) + return attention + + +def test_qwen_added_qkv_lossless_calls_a_wrapped_linear_as_is(): + torch.manual_seed(20260831) + dim = 64 + x = torch.randn(1, 17, dim, device="cuda", dtype=torch.bfloat16) + wrapped = _WrappedAddedQKV(dim) + attention = _added_qkv_attention(wrapped) + + actual = attention._get_added_qkv_projections(x) + expected = tuple( + tensor.contiguous() + for tensor in F.linear(x, wrapped.weight, wrapped.bias).chunk(3, dim=-1) + ) + assert wrapped.calls == 1 + assert all(torch.equal(a, e) for a, e in zip(actual, expected)) + + +def test_qwen_added_qkv_lossless_uses_three_reference_gemms(): + torch.manual_seed(20260831) + dim = 64 + x = torch.randn(1, 17, dim, device="cuda", dtype=torch.bfloat16) + packed = _PackedAddedQKV(dim) + attention = _added_qkv_attention(packed) expected_lossless = _split_unquantized_merged_linear(packed, x) actual_lossless = attention._get_added_qkv_projections(x) @@ -1551,5 +1595,69 @@ def test_qwen21_modulation_does_not_verify_during_capture(monkeypatch): assert torch.equal(out, norm(x) * (1 + scale)) +requires_nvidia_jit = pytest.mark.skipif( + not is_cuda(), reason="the Qwen-Image 2.1 JIT CUDA kernels are NVIDIA-only" +) + + +def _qwen21_rope(tokens: int, device: torch.device) -> torch.Tensor: + angles = torch.randn(tokens, 64, device=device, dtype=torch.float32) + return torch.polar(torch.ones_like(angles), angles) + + +@requires_nvidia_jit +@torch.no_grad() +def test_qwen21_cuda_qk_rope_verifies_and_preserves_native_fallback(monkeypatch): + x = torch.randn(1, 257, 8, 128, device="cuda", dtype=torch.bfloat16) + rope = _qwen21_rope(257, x.device) + norm = qwen_image21.RMSNorm( + 128, 1e-6, cast_x_before_out_mul=True, force_native=True + ).to(device=x.device, dtype=x.dtype) + norm.weight.normal_() + expected = qwen_image21.apply_rope(norm(x), rope) + gate = BitExactFusionGate("test CUDA Q/K norm + RoPE") + monkeypatch.setattr(qwen_image21, "_QK_ROPE_CUDA_FUSION", gate) + assert torch.equal(qwen_image21.apply_qk_norm_rope(x, norm, rope), expected) + assert gate.verified and not gate.disabled + x.normal_() + assert torch.equal( + qwen_image21.apply_qk_norm_rope(x, norm, rope), + qwen_image21.apply_rope(norm(x), rope), + ) + + gate = BitExactFusionGate("test mismatched CUDA Q/K norm + RoPE") + monkeypatch.setattr(qwen_image21, "_QK_ROPE_CUDA_FUSION", gate) + monkeypatch.setattr( + qwen_image21, + "qknorm_complex_rope_cuda", + lambda x, weight, rope, eps: torch.zeros_like(x), + ) + assert torch.equal( + qwen_image21.apply_qk_norm_rope(x, norm, rope), + qwen_image21.apply_rope(norm(x), rope), + ) + assert gate.disabled and not gate.verified + + +@requires_nvidia_jit +@torch.no_grad() +def test_qwen21_cuda_qk_rope_does_not_verify_during_capture(monkeypatch): + x = torch.randn(1, 17, 2, 128, device="cuda", dtype=torch.bfloat16) + rope = _qwen21_rope(17, x.device) + norm = qwen_image21.RMSNorm( + 128, 1e-6, cast_x_before_out_mul=True, force_native=True + ).to(device=x.device, dtype=x.dtype) + qwen_image21.apply_rope(norm(x), rope) + gate = BitExactFusionGate("test captured CUDA Q/K norm + RoPE") + monkeypatch.setattr(qwen_image21, "_QK_ROPE_CUDA_FUSION", gate) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + out = qwen_image21.apply_qk_norm_rope(x, norm, rope) + assert not gate.verified and not gate.disabled + x.normal_() + graph.replay() + assert torch.equal(out, qwen_image21.apply_rope(norm(x), rope)) + + if __name__ == "__main__": sys.exit(pytest.main([__file__, "-v"]))