[KDA+Kimi K3] Speed up LTX-2.3 QK norm and split RoPE on H200 (#41459)

Co-authored-by: kernel-design-agents <334018530+kernel-design-agents@users.noreply.github.com>
This commit is contained in:
Xiaoyu Zhang
2026-10-01 14:21:31 +08:00
committed by GitHub
co-authored by kernel-design-agents
parent 11500701f1
commit 77b64e353f
3 changed files with 992 additions and 5 deletions
@@ -0,0 +1,857 @@
// LTX2 Q/K RMSNorm + split RoPE for SM90.
// Fuse Q/K launches and use bf16x2 loads/stores for aligned 64/128-wide heads,
// preserving the original reduction order and intermediate BF16 rounding.
// Other head dimensions, unaligned pointers and odd RoPE strides use scalar
// loads/stores without narrowing the public input contract.
#pragma once
#include <sgl_kernel/tensor.h> // For TensorMatcher, SymbolicSize, SymbolicDevice
#include <sgl_kernel/utils.h> // For RuntimeCheck
#include <sgl_kernel/utils.cuh> // For LaunchKernel and CUDA dtype aliases
#include <cstdint>
#include <cuda_bf16.h>
#include <type_traits>
namespace sglang {
namespace ltx2_qknorm_split_rope_kda {
constexpr int kThreads = 128;
// Fuse Q/K below this row-count threshold. Larger grids and empty sides use
// separate launches, skipping any side with zero rows.
constexpr int64_t kSplitThreshold = KDA_SPLIT_THRESHOLD;
inline const char* data_ptr(const tvm::ffi::TensorView& t) {
return static_cast<const char*>(t.data_ptr()) + t.byte_offset();
}
inline char* mutable_data_ptr(const tvm::ffi::TensorView& t) {
return static_cast<char*>(t.data_ptr()) + t.byte_offset();
}
// Bit-exact copy of the pinned baseline sum-of-squares reduction (scalar bf16
// loads). Used by the scalar row path for any input that may be 2-byte
// aligned.
SGL_DEVICE float compute_rstd_scalar(
const bf16_t* __restrict__ xrow,
int64_t hidden_size,
float eps,
int tid,
int lane,
int warp_id,
float* warp_sum,
float* s_rstd) {
float local = 0.f;
const int64_t n_vec = hidden_size >> 2;
for (int64_t i = tid; i < n_vec; i += kThreads) {
const int64_t base = i << 2;
const float v0 = __bfloat162float(xrow[base + 0]);
const float v1 = __bfloat162float(xrow[base + 1]);
const float v2 = __bfloat162float(xrow[base + 2]);
const float v3 = __bfloat162float(xrow[base + 3]);
local = fmaf(v0, v0, local);
local = fmaf(v1, v1, local);
local = fmaf(v2, v2, local);
local = fmaf(v3, v3, local);
}
#pragma unroll
for (int offset = 16; offset > 0; offset >>= 1) {
local += __shfl_down_sync(0xffffffffu, local, offset);
}
if (lane == 0) {
warp_sum[warp_id] = local;
}
__syncthreads();
if (tid == 0) {
const float total = (warp_sum[0] + warp_sum[2]) + (warp_sum[1] + warp_sum[3]);
*s_rstd = rsqrtf(total / static_cast<float>(hidden_size) + eps);
}
__syncthreads();
return *s_rstd;
}
// bf16x2-vectorized sum-of-squares. Requires xrow to be 4-byte aligned (the
// host only calls this when x is 4-byte aligned and hidden_size % 4 == 0, so
// every 8-byte group read here is 4-byte aligned). The per-thread fmaf
// accumulation order is identical to the scalar version, so the reduction is
// bit-identical.
SGL_DEVICE float compute_rstd_vec(
const bf16_t* __restrict__ xrow,
int64_t hidden_size,
float eps,
int tid,
int lane,
int warp_id,
float* warp_sum,
float* s_rstd) {
float local = 0.f;
const int64_t n_vec2 = hidden_size >> 2; // groups of 4 elems = 2 bf16x2
for (int64_t i = tid; i < n_vec2; i += kThreads) {
const int64_t base = i << 2;
const __nv_bfloat162 p0 = reinterpret_cast<const __nv_bfloat162*>(xrow + base)[0];
const __nv_bfloat162 p1 = reinterpret_cast<const __nv_bfloat162*>(xrow + base)[1];
const float v0 = __bfloat162float(p0.x);
const float v1 = __bfloat162float(p0.y);
const float v2 = __bfloat162float(p1.x);
const float v3 = __bfloat162float(p1.y);
local = fmaf(v0, v0, local);
local = fmaf(v1, v1, local);
local = fmaf(v2, v2, local);
local = fmaf(v3, v3, local);
}
#pragma unroll
for (int offset = 16; offset > 0; offset >>= 1) {
local += __shfl_down_sync(0xffffffffu, local, offset);
}
if (lane == 0) {
warp_sum[warp_id] = local;
}
__syncthreads();
if (tid == 0) {
const float total = (warp_sum[0] + warp_sum[2]) + (warp_sum[1] + warp_sum[3]);
*s_rstd = rsqrtf(total / static_cast<float>(hidden_size) + eps);
}
__syncthreads();
return *s_rstd;
}
template <bool kRoundIntermediates>
SGL_DEVICE float norm_value(float x, float weight, float rstd) {
const float value = weight * (rstd * x);
if constexpr (kRoundIntermediates) {
return __bfloat162float(__float2bfloat16_rn(value));
}
return value;
}
template <bool kRoundIntermediates>
SGL_DEVICE void rope_pair(float x0, float x1, float cos, float sin, float& y0, float& y1) {
float p0 = x0 * cos;
float p1 = x1 * cos;
if constexpr (kRoundIntermediates) {
// Hopper eager stores the BF16 cosine product before its addcmul update.
p0 = __bfloat162float(__float2bfloat16_rn(p0));
p1 = __bfloat162float(__float2bfloat16_rn(p1));
}
y0 = fmaf(-sin, x1, p0);
y1 = fmaf(sin, x0, p1);
}
// Scalar row processing: bit-exact copy of the pinned baseline inner loop.
template <bool kRoundIntermediates>
SGL_DEVICE void process_row(
int64_t row,
const bf16_t* __restrict__ x,
const bf16_t* __restrict__ cos,
const bf16_t* __restrict__ sin,
const bf16_t* __restrict__ weight,
bf16_t* __restrict__ out,
float eps,
int64_t seq_len,
int64_t num_heads,
int64_t head_dim,
int64_t stride_cos_b,
int64_t stride_cos_h,
int64_t stride_cos_t,
int64_t stride_sin_b,
int64_t stride_sin_h,
int64_t stride_sin_t,
int tid,
int lane,
int warp_id,
float* warp_sum,
float* s_rstd) {
const int64_t batch = row / seq_len;
const int64_t token = row - batch * seq_len;
const int64_t hidden_size = num_heads * head_dim;
const int64_t half_dim = head_dim >> 1;
const auto* __restrict__ xrow = x + row * hidden_size;
auto* __restrict__ outrow = out + row * hidden_size;
const float rstd = compute_rstd_scalar(xrow, hidden_size, eps, tid, lane, warp_id, warp_sum, s_rstd);
const int64_t num_pairs = num_heads * half_dim;
for (int64_t pair = tid; pair < num_pairs; pair += kThreads) {
const int64_t head = pair / half_dim;
const int64_t offset = pair - head * half_dim;
const int64_t idx0 = head * head_dim + offset;
const int64_t idx1 = idx0 + half_dim;
const float n0 =
norm_value<kRoundIntermediates>(__bfloat162float(xrow[idx0]), __bfloat162float(weight[idx0]), rstd);
const float n1 =
norm_value<kRoundIntermediates>(__bfloat162float(xrow[idx1]), __bfloat162float(weight[idx1]), rstd);
const int64_t cos_offset = batch * stride_cos_b + head * stride_cos_h + token * stride_cos_t + offset;
const int64_t sin_offset = batch * stride_sin_b + head * stride_sin_h + token * stride_sin_t + offset;
float y0;
float y1;
rope_pair<kRoundIntermediates>(
n0, n1, __bfloat162float(cos[cos_offset]), __bfloat162float(sin[sin_offset]), y0, y1);
outrow[idx0] = __float2bfloat16_rn(y0);
outrow[idx1] = __float2bfloat16_rn(y1);
}
}
// bf16x2-vectorized row processing. Requires every dereferenced address to be
// 4-byte aligned; the host guarantees this via a runtime alignment/stride
// check before selecting this path (even head/token/batch cos/sin strides and
// 4-byte-aligned x, out, weight, cos, sin). head_dim is a compile-time
// template parameter for unrolling. Arithmetic is bit-identical to the scalar
// path.
template <bool kRoundIntermediates, int HEAD_DIM>
SGL_DEVICE void process_row_vec(
int64_t row,
const bf16_t* __restrict__ x,
const bf16_t* __restrict__ cos,
const bf16_t* __restrict__ sin,
const bf16_t* __restrict__ weight,
bf16_t* __restrict__ out,
float eps,
int64_t seq_len,
int64_t num_heads,
int64_t stride_cos_b,
int64_t stride_cos_h,
int64_t stride_cos_t,
int64_t stride_sin_b,
int64_t stride_sin_h,
int64_t stride_sin_t,
int tid,
int lane,
int warp_id,
float* warp_sum,
float* s_rstd) {
constexpr int64_t head_dim = HEAD_DIM;
constexpr int64_t half_dim = HEAD_DIM >> 1;
const int64_t batch = row / seq_len;
const int64_t token = row - batch * seq_len;
const int64_t hidden_size = num_heads * head_dim;
const auto* __restrict__ xrow = x + row * hidden_size;
auto* __restrict__ outrow = out + row * hidden_size;
const float rstd = compute_rstd_vec(xrow, hidden_size, eps, tid, lane, warp_id, warp_sum, s_rstd);
constexpr int64_t PAIRS_PER_HEAD = HEAD_DIM >> 2; // bf16x2 pairs per head
const int64_t num_vec = num_heads * PAIRS_PER_HEAD;
for (int64_t i = tid; i < num_vec; i += kThreads) {
const int64_t head = i / PAIRS_PER_HEAD;
const int64_t j = i - head * PAIRS_PER_HEAD; // pair index within head
const int64_t e0 = 2 * j;
const int64_t h = head * head_dim;
const int64_t idx0a = h + e0;
const int64_t idx1a = h + half_dim + e0;
const __nv_bfloat162 x0 = reinterpret_cast<const __nv_bfloat162*>(xrow + idx0a)[0];
const __nv_bfloat162 x1 = reinterpret_cast<const __nv_bfloat162*>(xrow + idx1a)[0];
const __nv_bfloat162 w0 = reinterpret_cast<const __nv_bfloat162*>(weight + idx0a)[0];
const __nv_bfloat162 w1 = reinterpret_cast<const __nv_bfloat162*>(weight + idx1a)[0];
const float n00 = norm_value<kRoundIntermediates>(__bfloat162float(x0.x), __bfloat162float(w0.x), rstd);
const float n01 = norm_value<kRoundIntermediates>(__bfloat162float(x0.y), __bfloat162float(w0.y), rstd);
const float n10 = norm_value<kRoundIntermediates>(__bfloat162float(x1.x), __bfloat162float(w1.x), rstd);
const float n11 = norm_value<kRoundIntermediates>(__bfloat162float(x1.y), __bfloat162float(w1.y), rstd);
const int64_t cs_base = batch * stride_cos_b + head * stride_cos_h + token * stride_cos_t;
const int64_t sin_base = batch * stride_sin_b + head * stride_sin_h + token * stride_sin_t;
const __nv_bfloat162 c = reinterpret_cast<const __nv_bfloat162*>(cos + cs_base + e0)[0];
const __nv_bfloat162 sn = reinterpret_cast<const __nv_bfloat162*>(sin + sin_base + e0)[0];
float y00, y01, y10, y11;
rope_pair<kRoundIntermediates>(n00, n10, __bfloat162float(c.x), __bfloat162float(sn.x), y00, y10);
rope_pair<kRoundIntermediates>(n01, n11, __bfloat162float(c.y), __bfloat162float(sn.y), y01, y11);
__nv_bfloat162 o0, o1;
o0.x = __float2bfloat16_rn(y00);
o0.y = __float2bfloat16_rn(y01);
o1.x = __float2bfloat16_rn(y10);
o1.y = __float2bfloat16_rn(y11);
// Stream the outputs, which are not read again by this kernel.
__stcs(reinterpret_cast<__nv_bfloat162*>(outrow + idx0a), o0);
__stcs(reinterpret_cast<__nv_bfloat162*>(outrow + idx1a), o1);
}
}
// Scalar single-side kernel (baseline-identical structure).
template <bool kRoundIntermediates>
__global__ void ltx2_qknorm_split_rope_kernel(
const bf16_t* __restrict__ x,
const bf16_t* __restrict__ cos,
const bf16_t* __restrict__ sin,
const bf16_t* __restrict__ weight,
bf16_t* __restrict__ out,
float eps,
int64_t seq_len,
int64_t num_heads,
int64_t head_dim,
int64_t stride_cos_b,
int64_t stride_cos_h,
int64_t stride_cos_t,
int64_t stride_sin_b,
int64_t stride_sin_h,
int64_t stride_sin_t) {
const int64_t row = static_cast<int64_t>(blockIdx.x);
const int tid = threadIdx.x + threadIdx.y * 32;
const int lane = threadIdx.x;
const int warp_id = threadIdx.y;
__shared__ float warp_sum[4];
__shared__ float s_rstd;
process_row<kRoundIntermediates>(
row,
x,
cos,
sin,
weight,
out,
eps,
seq_len,
num_heads,
head_dim,
stride_cos_b,
stride_cos_h,
stride_cos_t,
stride_sin_b,
stride_sin_h,
stride_sin_t,
tid,
lane,
warp_id,
warp_sum,
&s_rstd);
}
// Vectorized single-side kernel (used by the split path for one side).
template <bool kRoundIntermediates, int HEAD_DIM>
__global__ void ltx2_qknorm_split_rope_kernel_vec(
const bf16_t* __restrict__ x,
const bf16_t* __restrict__ cos,
const bf16_t* __restrict__ sin,
const bf16_t* __restrict__ weight,
bf16_t* __restrict__ out,
float eps,
int64_t seq_len,
int64_t num_heads,
int64_t stride_cos_b,
int64_t stride_cos_h,
int64_t stride_cos_t,
int64_t stride_sin_b,
int64_t stride_sin_h,
int64_t stride_sin_t) {
const int64_t row = static_cast<int64_t>(blockIdx.x);
const int tid = threadIdx.x + threadIdx.y * 32;
const int lane = threadIdx.x;
const int warp_id = threadIdx.y;
__shared__ float warp_sum[4];
__shared__ float s_rstd;
process_row_vec<kRoundIntermediates, HEAD_DIM>(
row,
x,
cos,
sin,
weight,
out,
eps,
seq_len,
num_heads,
stride_cos_b,
stride_cos_h,
stride_cos_t,
stride_sin_b,
stride_sin_h,
stride_sin_t,
tid,
lane,
warp_id,
warp_sum,
&s_rstd);
}
// Scalar fused q+k kernel (one launch, both sides, scalar inner loop).
template <bool kRoundIntermediates>
__global__ void ltx2_qknorm_split_rope_fused_kernel(
const bf16_t* __restrict__ q,
const bf16_t* __restrict__ q_cos,
const bf16_t* __restrict__ q_sin,
const bf16_t* __restrict__ q_weight,
bf16_t* __restrict__ q_out,
const bf16_t* __restrict__ k,
const bf16_t* __restrict__ k_cos,
const bf16_t* __restrict__ k_sin,
const bf16_t* __restrict__ k_weight,
bf16_t* __restrict__ k_out,
float eps,
int64_t q_rows,
int64_t q_seq_len,
int64_t k_seq_len,
int64_t num_heads,
int64_t head_dim,
int64_t q_stride_cos_b,
int64_t q_stride_cos_h,
int64_t q_stride_cos_t,
int64_t q_stride_sin_b,
int64_t q_stride_sin_h,
int64_t q_stride_sin_t,
int64_t k_stride_cos_b,
int64_t k_stride_cos_h,
int64_t k_stride_cos_t,
int64_t k_stride_sin_b,
int64_t k_stride_sin_h,
int64_t k_stride_sin_t) {
const int64_t row = static_cast<int64_t>(blockIdx.x);
const int tid = threadIdx.x + threadIdx.y * 32;
const int lane = threadIdx.x;
const int warp_id = threadIdx.y;
__shared__ float warp_sum[4];
__shared__ float s_rstd;
if (row < q_rows) {
process_row<kRoundIntermediates>(
row,
q,
q_cos,
q_sin,
q_weight,
q_out,
eps,
q_seq_len,
num_heads,
head_dim,
q_stride_cos_b,
q_stride_cos_h,
q_stride_cos_t,
q_stride_sin_b,
q_stride_sin_h,
q_stride_sin_t,
tid,
lane,
warp_id,
warp_sum,
&s_rstd);
} else {
process_row<kRoundIntermediates>(
row - q_rows,
k,
k_cos,
k_sin,
k_weight,
k_out,
eps,
k_seq_len,
num_heads,
head_dim,
k_stride_cos_b,
k_stride_cos_h,
k_stride_cos_t,
k_stride_sin_b,
k_stride_sin_h,
k_stride_sin_t,
tid,
lane,
warp_id,
warp_sum,
&s_rstd);
}
}
// Vectorized fused q+k kernel: q side may independently use the vectorized or
// scalar row path, and likewise for k (mixed support). The per-side choice is
// a template parameter selected by the host from runtime alignment checks.
template <bool kRoundIntermediates, int HEAD_DIM, bool Q_VEC, bool K_VEC>
__global__ void ltx2_qknorm_split_rope_fused_kernel_vec(
const bf16_t* __restrict__ q,
const bf16_t* __restrict__ q_cos,
const bf16_t* __restrict__ q_sin,
const bf16_t* __restrict__ q_weight,
bf16_t* __restrict__ q_out,
const bf16_t* __restrict__ k,
const bf16_t* __restrict__ k_cos,
const bf16_t* __restrict__ k_sin,
const bf16_t* __restrict__ k_weight,
bf16_t* __restrict__ k_out,
float eps,
int64_t q_rows,
int64_t q_seq_len,
int64_t k_seq_len,
int64_t num_heads,
int64_t q_stride_cos_b,
int64_t q_stride_cos_h,
int64_t q_stride_cos_t,
int64_t q_stride_sin_b,
int64_t q_stride_sin_h,
int64_t q_stride_sin_t,
int64_t k_stride_cos_b,
int64_t k_stride_cos_h,
int64_t k_stride_cos_t,
int64_t k_stride_sin_b,
int64_t k_stride_sin_h,
int64_t k_stride_sin_t) {
const int64_t row = static_cast<int64_t>(blockIdx.x);
const int tid = threadIdx.x + threadIdx.y * 32;
const int lane = threadIdx.x;
const int warp_id = threadIdx.y;
__shared__ float warp_sum[4];
__shared__ float s_rstd;
if (row < q_rows) {
if constexpr (Q_VEC) {
process_row_vec<kRoundIntermediates, HEAD_DIM>(
row,
q,
q_cos,
q_sin,
q_weight,
q_out,
eps,
q_seq_len,
num_heads,
q_stride_cos_b,
q_stride_cos_h,
q_stride_cos_t,
q_stride_sin_b,
q_stride_sin_h,
q_stride_sin_t,
tid,
lane,
warp_id,
warp_sum,
&s_rstd);
} else {
process_row<kRoundIntermediates>(
row,
q,
q_cos,
q_sin,
q_weight,
q_out,
eps,
q_seq_len,
num_heads,
HEAD_DIM,
q_stride_cos_b,
q_stride_cos_h,
q_stride_cos_t,
q_stride_sin_b,
q_stride_sin_h,
q_stride_sin_t,
tid,
lane,
warp_id,
warp_sum,
&s_rstd);
}
} else {
if constexpr (K_VEC) {
process_row_vec<kRoundIntermediates, HEAD_DIM>(
row - q_rows,
k,
k_cos,
k_sin,
k_weight,
k_out,
eps,
k_seq_len,
num_heads,
k_stride_cos_b,
k_stride_cos_h,
k_stride_cos_t,
k_stride_sin_b,
k_stride_sin_h,
k_stride_sin_t,
tid,
lane,
warp_id,
warp_sum,
&s_rstd);
} else {
process_row<kRoundIntermediates>(
row - q_rows,
k,
k_cos,
k_sin,
k_weight,
k_out,
eps,
k_seq_len,
num_heads,
HEAD_DIM,
k_stride_cos_b,
k_stride_cos_h,
k_stride_cos_t,
k_stride_sin_b,
k_stride_sin_h,
k_stride_sin_t,
tid,
lane,
warp_id,
warp_sum,
&s_rstd);
}
}
}
struct LTX2QKNormSplitRopeKernel {
template <bool kRoundIntermediates>
static void
run(tvm::ffi::TensorView q_out,
tvm::ffi::TensorView k_out,
tvm::ffi::TensorView q,
tvm::ffi::TensorView q_cos,
tvm::ffi::TensorView q_sin,
tvm::ffi::TensorView q_weight,
tvm::ffi::TensorView k,
tvm::ffi::TensorView k_cos,
tvm::ffi::TensorView k_sin,
tvm::ffi::TensorView k_weight,
double eps,
int64_t num_heads,
int64_t head_dim) {
using namespace host;
RuntimeCheck(num_heads > 0, "num_heads must be positive");
RuntimeCheck(head_dim > 0, "head_dim must be positive");
RuntimeCheck(head_dim % 2 == 0, "head_dim must be even");
const int64_t hidden_size = num_heads * head_dim;
RuntimeCheck(hidden_size % 4 == 0, "hidden size must be divisible by 4");
auto batch = SymbolicSize{"batch"};
auto q_seq_len = SymbolicSize{"q_seq_len"};
auto k_seq_len = SymbolicSize{"k_seq_len"};
auto heads = SymbolicSize{"num_heads"};
auto half_dim = SymbolicSize{"half_dim"};
auto device = SymbolicDevice{};
heads.set_value(num_heads);
half_dim.set_value(head_dim / 2);
device.set_options<kDLCUDA>();
TensorMatcher({batch, q_seq_len, hidden_size}).with_dtype<bf16_t>().with_device(device).verify(q).verify(q_out);
TensorMatcher({batch, k_seq_len, hidden_size}).with_dtype<bf16_t>().with_device(device).verify(k).verify(k_out);
TensorMatcher({hidden_size}).with_dtype<bf16_t>().with_device(device).verify(q_weight);
TensorMatcher({hidden_size}).with_dtype<bf16_t>().with_device(device).verify(k_weight);
TensorMatcher({batch, heads, q_seq_len, half_dim})
.with_strides({-1, -1, -1, 1})
.with_dtype<bf16_t>()
.with_device(device)
.verify(q_cos);
TensorMatcher({batch, heads, q_seq_len, half_dim})
.with_strides({-1, -1, -1, 1})
.with_dtype<bf16_t>()
.with_device(device)
.verify(q_sin);
TensorMatcher({batch, heads, k_seq_len, half_dim})
.with_strides({-1, -1, -1, 1})
.with_dtype<bf16_t>()
.with_device(device)
.verify(k_cos);
TensorMatcher({batch, heads, k_seq_len, half_dim})
.with_strides({-1, -1, -1, 1})
.with_dtype<bf16_t>()
.with_device(device)
.verify(k_sin);
const int64_t q_rows = batch.unwrap() * q_seq_len.unwrap();
const int64_t k_rows = batch.unwrap() * k_seq_len.unwrap();
const int64_t total_rows = q_rows + k_rows;
// Only global early return: nothing to do when both sides are empty. A
// single empty side is handled per-path below so the non-empty side is
// still computed (an empty side never selects the fused kernel).
if (total_rows == 0) {
return;
}
RuntimeCheck(total_rows <= static_cast<int64_t>(UINT32_MAX), "LTX2 QKNorm split-RoPE grid is too large");
const DLDevice dl_device = device.unwrap();
// Runtime eligibility for the bf16x2-vectorized row path, per side. Every
// bf16x2 address the kernel dereferences must be 4-byte aligned. With the
// guard's last-dim stride 1 and even head_dim this reduces to: the tensor
// base pointer is 4-byte aligned, and (for cos/sin) the head/token/batch
// strides are even so per-head offsets stay even. x/out/weight are
// contiguous with hidden_size % 4 == 0 and half_dim even, so their vector
// offsets are even whenever the base is 4-byte aligned.
const auto side_vec_ok = [&](const tvm::ffi::TensorView& x,
const tvm::ffi::TensorView& cos,
const tvm::ffi::TensorView& sin,
const tvm::ffi::TensorView& weight,
const tvm::ffi::TensorView& out) {
const bool base_aligned = (reinterpret_cast<uintptr_t>(data_ptr(x)) % 4 == 0) &&
(reinterpret_cast<uintptr_t>(data_ptr(cos)) % 4 == 0) &&
(reinterpret_cast<uintptr_t>(data_ptr(sin)) % 4 == 0) &&
(reinterpret_cast<uintptr_t>(data_ptr(weight)) % 4 == 0) &&
(reinterpret_cast<uintptr_t>(mutable_data_ptr(out)) % 4 == 0);
const bool strides_even = (cos.stride(0) % 2 == 0) && (cos.stride(1) % 2 == 0) && (cos.stride(2) % 2 == 0) &&
(sin.stride(0) % 2 == 0) && (sin.stride(1) % 2 == 0) && (sin.stride(2) % 2 == 0);
return base_aligned && strides_even;
};
const bool q_vec = side_vec_ok(q, q_cos, q_sin, q_weight, q_out);
const bool k_vec = side_vec_ok(k, k_cos, k_sin, k_weight, k_out);
// Use the single-side launch path when only one side has rows.
if (total_rows <= kSplitThreshold && q_rows > 0 && k_rows > 0) {
auto launch_vec = [&](auto head_dim_constant, auto q_vec_c, auto k_vec_c) {
constexpr int HD = decltype(head_dim_constant)::value;
constexpr bool QV = decltype(q_vec_c)::value;
constexpr bool KV = decltype(k_vec_c)::value;
LaunchKernel(dim3(static_cast<uint32_t>(total_rows)), dim3(32, 4), dl_device)(
ltx2_qknorm_split_rope_fused_kernel_vec<kRoundIntermediates, HD, QV, KV>,
reinterpret_cast<const bf16_t*>(data_ptr(q)),
reinterpret_cast<const bf16_t*>(data_ptr(q_cos)),
reinterpret_cast<const bf16_t*>(data_ptr(q_sin)),
reinterpret_cast<const bf16_t*>(data_ptr(q_weight)),
reinterpret_cast<bf16_t*>(mutable_data_ptr(q_out)),
reinterpret_cast<const bf16_t*>(data_ptr(k)),
reinterpret_cast<const bf16_t*>(data_ptr(k_cos)),
reinterpret_cast<const bf16_t*>(data_ptr(k_sin)),
reinterpret_cast<const bf16_t*>(data_ptr(k_weight)),
reinterpret_cast<bf16_t*>(mutable_data_ptr(k_out)),
static_cast<float>(eps),
q_rows,
q_seq_len.unwrap(),
k_seq_len.unwrap(),
num_heads,
q_cos.stride(0),
q_cos.stride(1),
q_cos.stride(2),
q_sin.stride(0),
q_sin.stride(1),
q_sin.stride(2),
k_cos.stride(0),
k_cos.stride(1),
k_cos.stride(2),
k_sin.stride(0),
k_sin.stride(1),
k_sin.stride(2));
};
auto launch_fused_scalar = [&]() {
LaunchKernel(dim3(static_cast<uint32_t>(total_rows)), dim3(32, 4), dl_device)(
ltx2_qknorm_split_rope_fused_kernel<kRoundIntermediates>,
reinterpret_cast<const bf16_t*>(data_ptr(q)),
reinterpret_cast<const bf16_t*>(data_ptr(q_cos)),
reinterpret_cast<const bf16_t*>(data_ptr(q_sin)),
reinterpret_cast<const bf16_t*>(data_ptr(q_weight)),
reinterpret_cast<bf16_t*>(mutable_data_ptr(q_out)),
reinterpret_cast<const bf16_t*>(data_ptr(k)),
reinterpret_cast<const bf16_t*>(data_ptr(k_cos)),
reinterpret_cast<const bf16_t*>(data_ptr(k_sin)),
reinterpret_cast<const bf16_t*>(data_ptr(k_weight)),
reinterpret_cast<bf16_t*>(mutable_data_ptr(k_out)),
static_cast<float>(eps),
q_rows,
q_seq_len.unwrap(),
k_seq_len.unwrap(),
num_heads,
head_dim,
q_cos.stride(0),
q_cos.stride(1),
q_cos.stride(2),
q_sin.stride(0),
q_sin.stride(1),
q_sin.stride(2),
k_cos.stride(0),
k_cos.stride(1),
k_cos.stride(2),
k_sin.stride(0),
k_sin.stride(1),
k_sin.stride(2));
};
if (head_dim == 64) {
if (q_vec && k_vec) {
launch_vec(std::integral_constant<int, 64>{}, std::true_type{}, std::true_type{});
} else if (q_vec) {
launch_vec(std::integral_constant<int, 64>{}, std::true_type{}, std::false_type{});
} else if (k_vec) {
launch_vec(std::integral_constant<int, 64>{}, std::false_type{}, std::true_type{});
} else {
launch_fused_scalar();
}
} else if (head_dim == 128) {
if (q_vec && k_vec) {
launch_vec(std::integral_constant<int, 128>{}, std::true_type{}, std::true_type{});
} else if (q_vec) {
launch_vec(std::integral_constant<int, 128>{}, std::true_type{}, std::false_type{});
} else if (k_vec) {
launch_vec(std::integral_constant<int, 128>{}, std::false_type{}, std::true_type{});
} else {
launch_fused_scalar();
}
} else {
launch_fused_scalar();
}
return;
}
// Large-grid or single-side path: separate launches. Each side independently uses the vectorized kernel when
// eligible, else the scalar kernel. Empty sides are skipped below.
auto launch_side = [&](const tvm::ffi::TensorView& x,
const tvm::ffi::TensorView& cos,
const tvm::ffi::TensorView& sin,
const tvm::ffi::TensorView& weight,
const tvm::ffi::TensorView& out,
int64_t rows,
int64_t seq,
bool vec_ok) {
if (rows == 0) {
return; // empty side: its output is already zero-sized, nothing to compute
}
if (vec_ok && (head_dim == 64 || head_dim == 128)) {
auto launch_vec_side = [&](auto head_dim_constant) {
constexpr int HD = decltype(head_dim_constant)::value;
LaunchKernel(dim3(static_cast<uint32_t>(rows)), dim3(32, 4), dl_device)(
ltx2_qknorm_split_rope_kernel_vec<kRoundIntermediates, HD>,
reinterpret_cast<const bf16_t*>(data_ptr(x)),
reinterpret_cast<const bf16_t*>(data_ptr(cos)),
reinterpret_cast<const bf16_t*>(data_ptr(sin)),
reinterpret_cast<const bf16_t*>(data_ptr(weight)),
reinterpret_cast<bf16_t*>(mutable_data_ptr(out)),
static_cast<float>(eps),
seq,
num_heads,
cos.stride(0),
cos.stride(1),
cos.stride(2),
sin.stride(0),
sin.stride(1),
sin.stride(2));
};
if (head_dim == 64) {
launch_vec_side(std::integral_constant<int, 64>{});
} else {
launch_vec_side(std::integral_constant<int, 128>{});
}
} else {
LaunchKernel(dim3(static_cast<uint32_t>(rows)), dim3(32, 4), dl_device)(
ltx2_qknorm_split_rope_kernel<kRoundIntermediates>,
reinterpret_cast<const bf16_t*>(data_ptr(x)),
reinterpret_cast<const bf16_t*>(data_ptr(cos)),
reinterpret_cast<const bf16_t*>(data_ptr(sin)),
reinterpret_cast<const bf16_t*>(data_ptr(weight)),
reinterpret_cast<bf16_t*>(mutable_data_ptr(out)),
static_cast<float>(eps),
seq,
num_heads,
head_dim,
cos.stride(0),
cos.stride(1),
cos.stride(2),
sin.stride(0),
sin.stride(1),
sin.stride(2));
}
};
launch_side(q, q_cos, q_sin, q_weight, q_out, q_rows, q_seq_len.unwrap(), q_vec);
launch_side(k, k_cos, k_sin, k_weight, k_out, k_rows, k_seq_len.unwrap(), k_vec);
}
};
} // namespace ltx2_qknorm_split_rope_kda
} // namespace sglang
@@ -4,7 +4,7 @@ from typing import TYPE_CHECKING
import torch
from sglang.kernels.jit.utils import cache_once, load_jit
from sglang.kernels.jit.utils import cache_once, get_jit_cuda_arch, load_jit
from sglang.kernels.kda_kernels import _cuda_source
from sglang.srt.utils.custom_op import register_custom_op
@@ -14,13 +14,23 @@ if TYPE_CHECKING:
@cache_once
def _jit_ltx2_qknorm_split_rope_module(round_intermediates: bool = False) -> Module:
arch = get_jit_cuda_arch()
sm90 = (arch.major, arch.minor) == (9, 0)
namespace = "ltx2_qknorm_split_rope_kda" if sm90 else "ltx2_qknorm_split_rope"
return load_jit(
"diffusion_ltx2_qknorm_split_rope",
cuda_files=[_cuda_source("diffusion/ltx2_qknorm_split_rope.cuh")],
cuda_files=[
_cuda_source(
"diffusion/ltx2_qknorm_split_rope_sm90.cuh"
if sm90
else "diffusion/ltx2_qknorm_split_rope.cuh"
)
],
extra_cuda_cflags=["-DKDA_SPLIT_THRESHOLD=1000000000"] if sm90 else [],
cuda_wrappers=[
(
"ltx2_qknorm_split_rope_pair",
"ltx2_qknorm_split_rope::LTX2QKNormSplitRopeKernel::run"
f"{namespace}::LTX2QKNormSplitRopeKernel::run"
f"<{str(round_intermediates).lower()}>",
)
],