[diffusion] move kernel validation into launchers and simplify dispatch (#42391)

This commit is contained in:
Xiaoyu Zhang
2026-10-04 18:23:29 +08:00
committed by GitHub
parent 29bcb74a89
commit bc5a055dd7
74 changed files with 894 additions and 1534 deletions
@@ -52,25 +52,33 @@ __global__ void gelu_tanh_cat_kernel(
template <typename T>
void gelu_tanh_cat(tvm::ffi::TensorView attn, tvm::ffi::TensorView mlp, tvm::ffi::TensorView output) {
using namespace host;
auto rows = SymbolicSize{"rows"};
auto attn_width = SymbolicSize{"attn_width"};
auto mlp_width = SymbolicSize{"mlp_width"};
CHECK_HOST(attn.ndim() >= 2) << "gelu_tanh_cat: expected at least two dimensions";
auto dtype = SymbolicDType{};
dtype.set_options<T>();
auto device_ = SymbolicDevice{};
device_.set_options<kDLCUDA>();
TensorMatcher({rows, attn_width}).with_dtype<T>().with_device(device_).verify(attn);
TensorMatcher({rows, mlp_width}).with_dtype<T>().with_device(device_).verify(mlp);
const int64_t a = attn_width.unwrap();
const int64_t m = mlp_width.unwrap();
TensorMatcher({rows, a + m}).with_dtype<T>().with_device(device_).verify(output);
for (const auto tensor : {attn, mlp, output}) {
dtype.verify(tensor.dtype());
device_.verify(tensor.device());
CHECK_HOST(tensor.ndim() == attn.ndim() && tensor.is_contiguous())
<< "gelu_tanh_cat: expected contiguous tensors of equal rank";
for (int64_t dim = 0; dim < attn.ndim() - 1; ++dim) {
CHECK_HOST(tensor.size(dim) == attn.size(dim)) << "gelu_tanh_cat: leading dimensions must match";
}
}
const int64_t a = attn.size(-1);
const int64_t m = mlp.size(-1);
CHECK_HOST(output.size(-1) == a + m) << "gelu_tanh_cat: incorrect output width";
constexpr int kVecN = 16 / sizeof(T);
CHECK_HOST(a > 0 && m > 0 && a % kVecN == 0 && m % kVecN == 0)
<< "gelu_tanh_cat: both widths must be positive multiples of " << kVecN;
CHECK_HOST(rows.unwrap() > 0) << "gelu_tanh_cat: rows must be positive";
const int64_t rows = attn.numel() / a;
CHECK_HOST(rows > 0) << "gelu_tanh_cat: rows must be positive";
CHECK_HOST(
reinterpret_cast<uintptr_t>(attn.data_ptr()) % 16 == 0 && reinterpret_cast<uintptr_t>(mlp.data_ptr()) % 16 == 0 &&
reinterpret_cast<uintptr_t>(output.data_ptr()) % 16 == 0)
<< "gelu_tanh_cat: tensors must be 16-byte aligned";
const int64_t n = rows.unwrap() * ((a + m) / kVecN);
const int64_t n = rows * ((a + m) / kVecN);
CHECK_HOST(n <= std::numeric_limits<int32_t>::max()) << "gelu_tanh_cat: tensor is too large";
constexpr int kBlockSize = 256;
const auto kernel = gelu_tanh_cat_kernel<T, kVecN>;
@@ -19,7 +19,7 @@ namespace sglang {
* multiply and add/subtract operations preserve the separate eager FP32
* intermediates before the result is rounded back to fp16/bf16.
*
* \tparam T Activation type: fp16_t or bf16_t
* \tparam T Activation type: fp16_t, bf16_t or fp32_t
* \param q Normalized query tensor, contiguous [tokens, heads, head_dim]
* \param k Normalized key tensor, contiguous [tokens, heads, head_dim]
* \param freqs Transposed Helios frequency tensor, contiguous
@@ -38,7 +38,7 @@ __global__ void helios_qk_rope_kernel(
uint32_t pairs_per_head,
uint32_t num_heads,
uint32_t freq_stride) {
static_assert(std::is_same_v<T, fp16_t> || std::is_same_v<T, bf16_t>);
static_assert(std::is_same_v<T, fp16_t> || std::is_same_v<T, bf16_t> || std::is_same_v<T, fp32_t>);
using Packed = packed_t<T>;
auto* q_pairs = reinterpret_cast<Packed*>(q);
@@ -79,10 +79,18 @@ struct HeliosQKRoPEKernel {
auto device = SymbolicDevice{};
device.set_options<kDLCUDA>();
TensorMatcher({N, H, D}).with_dtype<DType>().with_device(device).verify(q).verify(k);
TensorMatcher({N, F}).with_dtype<fp32_t>().with_device(device).verify(freqs);
int64_t batch = 1;
if (q.ndim() == 4) {
auto B = SymbolicSize{"batch"};
TensorMatcher({B, N, H, D}).with_dtype<DType>().with_device(device).verify(q).verify(k);
TensorMatcher({B, N, F}).with_dtype<fp32_t>().with_device(device).verify(freqs);
batch = B.unwrap();
} else {
TensorMatcher({N, H, D}).with_dtype<DType>().with_device(device).verify(q).verify(k);
TensorMatcher({N, F}).with_dtype<fp32_t>().with_device(device).verify(freqs);
}
const int64_t tokens = N.unwrap();
const int64_t tokens = batch * N.unwrap();
const int64_t heads = H.unwrap();
const int64_t head_dim = D.unwrap();
const int64_t freq_dim = F.unwrap();
@@ -91,27 +91,27 @@ struct InterleavedRopeFP64Kernel {
tvm::ffi::TensorView q,
tvm::ffi::TensorView k,
tvm::ffi::TensorView cos,
tvm::ffi::TensorView sin,
int64_t batch_size,
int64_t seq_len,
int64_t num_heads,
int64_t head_dim) {
tvm::ffi::TensorView sin) {
using namespace host;
using Pair = device::AlignedVector<T, 2>;
auto N = SymbolicSize{"activation_elements"};
auto R = SymbolicSize{"table_elements"};
auto B = SymbolicSize{"batch"};
auto S = SymbolicSize{"sequence"};
auto H = SymbolicSize{"heads"};
auto D = SymbolicSize{"head_dim"};
auto device = SymbolicDevice{};
device.set_options<kDLCUDA>();
TensorMatcher({N}).with_dtype<T>().with_device(device).verify(q_out).verify(k_out).verify(q).verify(k);
TensorMatcher({R}).with_dtype<double>().with_device(device).verify(cos).verify(sin);
TensorMatcher({B, S, H, D}).with_dtype<T>().with_device(device).verify(q_out).verify(k_out).verify(q).verify(k);
TensorMatcher({1, S, 1, D}).with_dtype<double>().with_device(device).verify(cos).verify(sin);
const int64_t batch_size = B.unwrap();
const int64_t seq_len = S.unwrap();
const int64_t num_heads = H.unwrap();
const int64_t head_dim = D.unwrap();
CHECK_HOST(batch_size > 0 && seq_len > 0 && num_heads > 0 && head_dim > 0)
<< "interleaved_rope_fp64 dimensions must be positive";
CHECK_HOST(head_dim % 2 == 0) << "interleaved_rope_fp64 head_dim must be even";
CHECK_HOST(N.unwrap() == batch_size * seq_len * num_heads * head_dim)
<< "interleaved_rope_fp64 activation shape does not match dimensions";
CHECK_HOST(R.unwrap() == seq_len * head_dim) << "interleaved_rope_fp64 table shape does not match dimensions";
CHECK_HOST(
q_out.data_ptr() != k_out.data_ptr() && q_out.data_ptr() != q.data_ptr() && q_out.data_ptr() != k.data_ptr() &&
k_out.data_ptr() != q.data_ptr() && k_out.data_ptr() != k.data_ptr())
@@ -123,7 +123,7 @@ struct InterleavedRopeFP64Kernel {
reinterpret_cast<uintptr_t>(k.data_ptr()) % alignof(Pair) == 0)
<< "interleaved_rope_fp64 activations must be aligned to rotation pairs";
const int64_t num_pairs = N.unwrap() / 2;
const int64_t num_pairs = q.numel() / 2;
const int64_t pairs_per_head = head_dim / 2;
const auto blocks =
static_cast<uint32_t>(std::min<int64_t>(div_ceil(num_pairs, static_cast<int64_t>(kBlockSize)), kMaxGrid));
@@ -122,27 +122,32 @@ struct LTX25DecoderRopeKernel {
tvm::ffi::TensorView sin_h,
tvm::ffi::TensorView cos_w,
tvm::ffi::TensorView sin_w,
int64_t batch_size,
int64_t num_frames,
int64_t height,
int64_t width,
int64_t num_heads,
int64_t head_dim,
int64_t dim_t,
int64_t dim_h) {
using namespace host;
using Pair = device::AlignedVector<T, 2>;
auto N = SymbolicSize{"activation_elements"};
auto RT = SymbolicSize{"temporal_table_elements"};
auto RH = SymbolicSize{"height_table_elements"};
auto RW = SymbolicSize{"width_table_elements"};
auto B = SymbolicSize{"batch"};
auto F = SymbolicSize{"frames"};
auto H = SymbolicSize{"height"};
auto W = SymbolicSize{"width"};
auto A = SymbolicSize{"heads"};
auto D = SymbolicSize{"head_dim"};
auto device = SymbolicDevice{};
device.set_options<kDLCUDA>();
TensorMatcher({N}).with_dtype<T>().with_device(device).verify(q_out).verify(k_out).verify(q).verify(k);
TensorMatcher({RT}).with_dtype<float>().with_device(device).verify(cos_t).verify(sin_t);
TensorMatcher({RH}).with_dtype<float>().with_device(device).verify(cos_h).verify(sin_h);
TensorMatcher({RW}).with_dtype<float>().with_device(device).verify(cos_w).verify(sin_w);
TensorMatcher({B, F, H, W, A, D})
.with_dtype<T>()
.with_device(device)
.verify(q_out)
.verify(k_out)
.verify(q)
.verify(k);
const int64_t batch_size = B.unwrap();
const int64_t num_frames = F.unwrap();
const int64_t height = H.unwrap();
const int64_t width = W.unwrap();
const int64_t num_heads = A.unwrap();
const int64_t head_dim = D.unwrap();
CHECK_HOST(batch_size > 0 && num_frames > 0 && height > 0 && width > 0 && num_heads > 0 && head_dim > 0)
<< "ltx25_decoder_rope dimensions must be positive";
@@ -150,11 +155,9 @@ struct LTX25DecoderRopeKernel {
<< "ltx25_decoder_rope dimensions must split into positive rotation pairs";
const int64_t dim_w = head_dim - dim_t - dim_h;
CHECK_HOST(dim_w > 0 && dim_w % 2 == 0) << "ltx25_decoder_rope width dimension must be positive and even";
CHECK_HOST(N.unwrap() == batch_size * num_frames * height * width * num_heads * head_dim)
<< "ltx25_decoder_rope activation shape does not match dimensions";
CHECK_HOST(RT.unwrap() == num_frames * dim_t / 2) << "ltx25_decoder_rope temporal table shape mismatch";
CHECK_HOST(RH.unwrap() == height * dim_h / 2) << "ltx25_decoder_rope height table shape mismatch";
CHECK_HOST(RW.unwrap() == width * dim_w / 2) << "ltx25_decoder_rope width table shape mismatch";
TensorMatcher({F, dim_t / 2}).with_dtype<float>().with_device(device).verify(cos_t).verify(sin_t);
TensorMatcher({H, dim_h / 2}).with_dtype<float>().with_device(device).verify(cos_h).verify(sin_h);
TensorMatcher({W, dim_w / 2}).with_dtype<float>().with_device(device).verify(cos_w).verify(sin_w);
CHECK_HOST(
q_out.data_ptr() != k_out.data_ptr() && q_out.data_ptr() != q.data_ptr() && q_out.data_ptr() != k.data_ptr() &&
k_out.data_ptr() != q.data_ptr() && k_out.data_ptr() != k.data_ptr())
@@ -166,7 +169,7 @@ struct LTX25DecoderRopeKernel {
reinterpret_cast<uintptr_t>(k.data_ptr()) % alignof(Pair) == 0)
<< "ltx25_decoder_rope activations must be aligned to rotation pairs";
const int64_t num_pairs = N.unwrap() / 2;
const int64_t num_pairs = q.numel() / 2;
const int64_t pairs_per_head = head_dim / 2;
const auto blocks =
static_cast<uint32_t>(std::min<int64_t>(div_ceil(num_pairs, static_cast<int64_t>(kBlockSize)), kMaxGrid));
@@ -211,7 +211,8 @@ __global__ void kernel(const Params __grid_constant__ params) {
template <bool kFuseRealSin, bool kCopyV>
struct Kernel {
/// \brief Single tensor, out of place: `out = rope(rmsnorm(x))` for a contiguous [B, S, H, 128] bf16 tensor.
/// \brief Single tensor, out of place: `out = rope(rmsnorm(x))` for a [B, S, H, 128] bf16 tensor with packed head
/// rows.
static void
run(tvm::ffi::TensorView x,
tvm::ffi::TensorView out,
@@ -224,19 +225,31 @@ struct Kernel {
auto H = SymbolicSize{"heads"};
auto device = SymbolicDevice{};
device.set_options<kDLCUDA>();
auto Bx = SymbolicSize{"input_batch_stride"};
auto Tx = SymbolicSize{"input_token_stride"};
TensorMatcher({B, S, H, kHeadDim})
.with_strides({Bx, Tx, kHeadDim, 1})
.with_dtype<bf16_t>()
.with_device(device)
.ensure_alignment(kVecAlignBytes)
.verify(x);
TensorMatcher({B, S, H, kHeadDim})
.with_dtype<bf16_t>()
.with_device(device)
.ensure_alignment(kVecAlignBytes)
.verify(x)
.verify(out);
TensorMatcher({kHeadDim}).with_dtype<bf16_t>().with_device(device).verify(weight);
TensorMatcher({S, kHeadDim}).with_dtype<fp32_t>().with_device(device).ensure_alignment(16).verify(rope);
TensorMatcher({S, kHeadDim / 2, 2}).with_dtype<fp32_t>().with_device(device).verify(rope);
// float4 loads span two complex pairs; only the token rows need 16-byte alignment.
CHECK_HOST(reinterpret_cast<uintptr_t>(rope.data_ptr()) % 16 == 0) << "RoPE cache must be 16-byte aligned";
CHECK_HOST(
Bx.unwrap() % (kVecAlignBytes / sizeof(bf16_t)) == 0 && Tx.unwrap() % (kVecAlignBytes / sizeof(bf16_t)) == 0)
<< "input batch/token strides must preserve vector alignment";
const int64_t batch = B.unwrap(), seq = S.unwrap(), heads = H.unwrap();
if (batch == 0 || seq == 0 || heads == 0) return;
const int64_t token_stride = heads * kHeadDim;
Params params{};
params.q_src = Rows{static_cast<const bf16_t*>(x.data_ptr()), seq * token_stride, token_stride};
params.q_src = Rows{static_cast<const bf16_t*>(x.data_ptr()), Bx.unwrap(), Tx.unwrap()};
params.q_dst = MutableRows{static_cast<bf16_t*>(out.data_ptr()), seq * token_stride, token_stride};
params.q_weight = static_cast<const bf16_t*>(weight.data_ptr());
params.k_weight = params.q_weight;
@@ -301,7 +314,9 @@ struct Kernel {
.verify(k_prefix)
.verify(v_prefix);
TensorMatcher({kHeadDim}).with_dtype<bf16_t>().with_device(device).verify(q_weight).verify(k_weight);
TensorMatcher({S, kHeadDim}).with_dtype<fp32_t>().with_device(device).ensure_alignment(16).verify(rope);
TensorMatcher({S, kHeadDim / 2, 2}).with_dtype<fp32_t>().with_device(device).verify(rope);
// float4 loads span two complex pairs; only the token rows need 16-byte alignment.
CHECK_HOST(reinterpret_cast<uintptr_t>(rope.data_ptr()) % 16 == 0) << "RoPE cache must be 16-byte aligned";
const int64_t batch = B.unwrap(), seq = S.unwrap(), heads = H.unwrap(), prefix = P.unwrap();
CHECK_HOST(PS.unwrap() == prefix + seq) << "k_out/v_out must hold prefix + seq tokens";
if (batch == 0 || seq == 0 || heads == 0) return;
@@ -185,11 +185,11 @@ SGL_DEVICE void step(float2 (&t)[8][4], Smem& sm, int kt, int ti, int tj, int i0
/**
* \brief transition = diag(alpha) (I + A)^-1, injection = B (I + A)^-1 for a batch of 128x128 SPD A.
*
* \param A [N, 128, 128] fp32, symmetric positive semi-definite (I + A is inverted)
* \param B [N, 128, 128] fp32
* \param alpha [N, 128] fp32 row scales of the transition
* \param transition [N, 128, 128] fp32 output
* \param injection [N, 128, 128] fp32 output
* \param A [..., 128, 128] fp32, symmetric positive semi-definite (I + A is inverted)
* \param B [..., 128, 128] fp32
* \param alpha [..., 128] fp32 row scales of the transition
* \param transition [..., 128, 128] fp32 output
* \param injection [..., 128, 128] fp32 output
*/
__global__ void __launch_bounds__(kBlockSize, kMinBlocksPerSm) vdn_delta_factors_kernel(
const float* __restrict__ A,
@@ -305,11 +305,11 @@ struct VdnDeltaFactorsKernel {
/**
* \brief Validate the tensors and launch one CTA per matrix.
*
* \param transition [N, 128, 128] fp32 output, diag(alpha) (I + A)^-1
* \param injection [N, 128, 128] fp32 output, B (I + A)^-1
* \param A [N, 128, 128] fp32 SPD statistics (I + A is inverted)
* \param B [N, 128, 128] fp32
* \param alpha [N, 128] fp32
* \param transition [..., 128, 128] fp32 output, diag(alpha) (I + A)^-1
* \param injection [..., 128, 128] fp32 output, B (I + A)^-1
* \param A [..., 128, 128] fp32 SPD statistics (I + A is inverted)
* \param B [..., 128, 128] fp32
* \param alpha [..., 128] fp32
*/
static void
run(tvm::ffi::TensorView transition,
@@ -318,18 +318,28 @@ struct VdnDeltaFactorsKernel {
tvm::ffi::TensorView B,
tvm::ffi::TensorView alpha) {
using namespace host;
auto N = SymbolicSize{"num_matrices"};
CHECK_HOST(A.ndim() >= 2 && A.size(-2) == kDim && A.size(-1) == kDim)
<< "vdn_delta_factors expects [..., 128, 128] matrices";
auto dtype = SymbolicDType{};
dtype.set_options<fp32_t>();
auto device = SymbolicDevice{};
device.set_options<kDLCUDA>();
TensorMatcher({N, kDim, kDim})
.with_dtype<fp32_t>()
.with_device(device)
.verify(transition)
.verify(injection)
.verify(A)
.verify(B);
TensorMatcher({N, kDim}).with_dtype<fp32_t>().with_device(device).verify(alpha);
const int64_t num = N.unwrap();
for (const auto tensor : {A, B, transition, injection, alpha}) {
dtype.verify(tensor.dtype());
device.verify(tensor.device());
CHECK_HOST(tensor.is_contiguous()) << "vdn_delta_factors expects contiguous tensors";
}
for (const auto tensor : {B, transition, injection}) {
CHECK_HOST(tensor.ndim() == A.ndim()) << "vdn_delta_factors matrix ranks must match";
for (int64_t dim = 0; dim < A.ndim(); ++dim) {
CHECK_HOST(tensor.size(dim) == A.size(dim)) << "vdn_delta_factors matrix shapes must match";
}
}
CHECK_HOST(alpha.ndim() == A.ndim() - 1) << "alpha must have shape A.shape[:-1]";
for (int64_t dim = 0; dim < alpha.ndim(); ++dim) {
CHECK_HOST(alpha.size(dim) == A.size(dim)) << "alpha must have shape A.shape[:-1]";
}
const int64_t num = A.numel() / (kDim * kDim);
if (num == 0) return;
CHECK_HOST(
transition.data_ptr() != A.data_ptr() && transition.data_ptr() != B.data_ptr() &&
@@ -17,6 +17,8 @@ _SUPPORTED_DTYPES = (torch.float16, torch.bfloat16, torch.float32)
@cache_once
def _jit_causal_conv3d_cat_pad_module(dtype: torch.dtype) -> Module:
if dtype not in _SUPPORTED_DTYPES:
raise RuntimeError(f"unsupported dtype for causal Conv3D cat/pad: {dtype}")
args = make_cpp_args(dtype)
return load_jit(
"diffusion_causal_conv3d_cat_pad",
@@ -101,19 +103,7 @@ def fused_causal_conv3d_cat_pad_cuda(
cache_x: torch.Tensor,
padding: list[int] | tuple[int, ...],
) -> torch.Tensor:
if x.dtype not in _SUPPORTED_DTYPES:
raise RuntimeError(f"unsupported dtype for causal Conv3D cat/pad: {x.dtype}")
if not torch.compiler.is_compiling():
if (
not x.is_cuda
or not cache_x.is_cuda
or x.dim() != 5
or cache_x.dim() != 5
or not x.is_contiguous()
or not cache_x.is_contiguous()
or not can_use_fused_causal_conv3d_cat_pad_cuda(x, cache_x, padding)
):
raise RuntimeError("unsupported input for causal Conv3D cat/pad CUDA")
"""Concatenate the temporal cache and pad; the C++ launcher validates inputs."""
return _causal_conv3d_cat_pad_custom_op(x, cache_x, *padding)
@@ -26,8 +26,6 @@ constexpr uint32_t kBroadcastRowsPerBlock = 4;
constexpr uint32_t kBroadcastColsPerBlock = 256;
constexpr uint32_t kMaxGrid = 65535;
constexpr uintptr_t kAlignment = 16;
constexpr uint32_t kTransposeTile = 32;
constexpr uint32_t kTransposeBlockSize = 256;
enum class GateMode : int { kFull, kBroadcastRow, kPerToken };
@@ -116,55 +114,6 @@ __global__ void residual_gate_add_scalar_kernel(
}
}
/**
* Fuse the residual-gate update when the residual/output use the common
* ``[batch, hidden, tokens]`` backing layout exposed as ``[batch, tokens,
* hidden]``, while update remains contiguous in the logical layout.
*
* Loading update in logical row-major order and consuming it transposed from
* shared memory keeps both update reads and residual/output traffic coalesced.
*/
template <typename T>
__global__ void residual_gate_add_transposed_kernel(
T* __restrict__ out,
const T* __restrict__ residual,
const T* __restrict__ update,
const T* __restrict__ gate,
int64_t tokens,
int64_t hidden_size) {
__shared__ T update_tile[kTransposeTile][kTransposeTile + 1];
const int64_t batch = blockIdx.z;
const int64_t token_base = static_cast<int64_t>(blockIdx.x) * kTransposeTile;
const int64_t hidden_base = static_cast<int64_t>(blockIdx.y) * kTransposeTile;
const int64_t batch_offset = batch * tokens * hidden_size;
#pragma unroll
for (uint32_t item = threadIdx.x; item < kTransposeTile * kTransposeTile; item += kTransposeBlockSize) {
const uint32_t token_in_tile = item / kTransposeTile;
const uint32_t hidden_in_tile = item % kTransposeTile;
const int64_t token = token_base + token_in_tile;
const int64_t hidden = hidden_base + hidden_in_tile;
if (token < tokens && hidden < hidden_size) {
update_tile[token_in_tile][hidden_in_tile] = update[batch_offset + token * hidden_size + hidden];
}
}
__syncthreads();
#pragma unroll
for (uint32_t item = threadIdx.x; item < kTransposeTile * kTransposeTile; item += kTransposeBlockSize) {
const uint32_t hidden_in_tile = item / kTransposeTile;
const uint32_t token_in_tile = item % kTransposeTile;
const int64_t token = token_base + token_in_tile;
const int64_t hidden = hidden_base + hidden_in_tile;
if (token < tokens && hidden < hidden_size) {
const int64_t transposed_offset = batch_offset + hidden * tokens + token;
out[transposed_offset] = residual_gate_value(
residual[transposed_offset], update_tile[token_in_tile][hidden_in_tile], SGLANG_LDG(gate + hidden));
}
}
}
} // namespace
/**
@@ -281,49 +230,6 @@ struct ResidualGateAddKernel {
hidden_size);
}
}
static void run_transposed(
tvm::ffi::TensorView out, tvm::ffi::TensorView residual, tvm::ffi::TensorView update, tvm::ffi::TensorView gate) {
using namespace host;
auto B = SymbolicSize{"batch"};
auto S = SymbolicSize{"tokens"};
auto D = SymbolicSize{"hidden_size"};
auto device = SymbolicDevice{};
device.set_options<kDLCUDA>();
TensorMatcher({B, S, D}).with_strides({-1, 1, -1}).with_dtype<T>().with_device(device).verify(out).verify(residual);
TensorMatcher({B, S, D}).with_strides({-1, -1, 1}).with_dtype<T>().with_device(device).verify(update);
TensorMatcher({1, 1, D}).with_strides({-1, -1, 1}).with_dtype<T>().with_device(device).verify(gate);
const int64_t batch = B.unwrap();
const int64_t tokens = S.unwrap();
const int64_t hidden_size = D.unwrap();
CHECK_HOST(batch > 0 && tokens > 0 && hidden_size > 0) << "transposed residual-gate tensors must be non-empty";
CHECK_HOST(
batch <= kMaxGrid && div_ceil(tokens, int64_t{kTransposeTile}) <= kMaxGrid &&
div_ceil(hidden_size, int64_t{kTransposeTile}) <= kMaxGrid)
<< "transposed residual-gate grid exceeds CUDA limits";
const int64_t batch_stride = tokens * hidden_size;
CHECK_HOST(
out.stride(0) == batch_stride && out.stride(2) == tokens && residual.stride(0) == batch_stride &&
residual.stride(2) == tokens)
<< "residual/output must use the transposed dense layout";
CHECK_HOST(update.stride(0) == batch_stride && update.stride(1) == hidden_size) << "update must be contiguous";
auto* out_ptr = static_cast<T*>(out.data_ptr());
const auto* residual_ptr = static_cast<const T*>(residual.data_ptr());
const auto* update_ptr = static_cast<const T*>(update.data_ptr());
const auto* gate_ptr = static_cast<const T*>(gate.data_ptr());
CHECK_HOST(out_ptr != residual_ptr && out_ptr != update_ptr && out_ptr != gate_ptr)
<< "output must not alias an input";
const dim3 grid(
static_cast<uint32_t>(div_ceil(tokens, int64_t{kTransposeTile})),
static_cast<uint32_t>(div_ceil(hidden_size, int64_t{kTransposeTile})),
static_cast<uint32_t>(batch));
LaunchKernel(grid, kTransposeBlockSize, device.unwrap())(
residual_gate_add_transposed_kernel<T>, out_ptr, residual_ptr, update_ptr, gate_ptr, tokens, hidden_size);
}
};
} // namespace residual_gate_add
@@ -402,33 +402,45 @@ def _qk_head_launch_config() -> tuple[int, int]:
def _mod_row_stride(t: torch.Tensor, batch: int, hidden: int) -> int | None:
# (batch, hidden) modulation rows, possibly strided views of a chunked
# adaLN projection; the last dim must be packed.
if t.dim() != 2 or t.shape != (batch, hidden) or t.stride(1) != 1:
if t.shape != (batch, hidden) or t.stride(1) != 1:
return None
return t.stride(0) if batch > 1 else hidden
def can_use_fused_layernorm_modulate(
x: torch.Tensor, scale: torch.Tensor, shift: torch.Tensor | None
) -> bool:
def can_use_fused_layernorm_modulate(dtype: torch.dtype, hidden: int) -> bool:
"""Select the BF16 vectorized LayerNorm reduction replicated below."""
return (
is_cuda() and dtype is torch.bfloat16 and 0 < hidden <= 8192 and hidden % 4 == 0
)
def _validate_layernorm_modulate(x, scale, shift) -> tuple[int, int, int, int]:
shape = x.shape
if not (
_is_bf16_cuda(x)
and x.dim() == 3
and len(shape) == 3
and x.numel() > 0
and x.is_contiguous()
and x.shape[-1] % 4 == 0
and x.shape[-1] <= 8192
and _is_bf16_cuda(scale)
and scale.device == x.device
and 0 < shape[-1] <= 8192
and shape[-1] % 4 == 0
):
return False
batch, _, hidden = x.shape
q = _mod_row_stride(scale, batch, hidden)
if shift is None:
return q is not None
if not _is_bf16_cuda(shift) or shift.device != x.device:
return False
v = _mod_row_stride(shift, batch, hidden)
return q is not None and v is not None and q == v
raise RuntimeError(
"LayerNorm modulation expects contiguous BF16 CUDA [B, S, D], D divisible by 4 and <= 8192"
)
batch, seq_len, hidden = shape
device = x.device
stride = _mod_row_stride(scale, batch, hidden)
if not (
stride is not None and scale.dtype is torch.bfloat16 and scale.device == device
):
raise RuntimeError("scale must have packed BF16 [B, D] rows on x's device")
if shift is not None and not (
shift.dtype is torch.bfloat16
and shift.device == device
and _mod_row_stride(shift, batch, hidden) == stride
):
raise RuntimeError("shift must match scale's row layout and dtype/device")
return batch, seq_len, hidden, stride
def _fake_ln_modulate(
@@ -510,11 +522,10 @@ def fused_layernorm_modulate_raw(
of microseconds per call); use it on CPU-launch-bound eager hot paths
(e.g. Sana), and the registered custom op under ``torch.compile``.
"""
batch, seq_len, hidden = x.shape
batch, seq_len, hidden, stride = _validate_layernorm_modulate(x, scale, shift)
n_rows = batch * seq_len
rows = 2
out = torch.empty_like(x)
stride = _mod_row_stride(scale, batch, hidden)
with torch.cuda.device(x.device):
_layernorm_modulate_kernel[(triton.cdiv(n_rows, rows),)](
out,
@@ -558,11 +569,10 @@ def fused_layernorm_modulate_fp8_quant_raw(
Unlike that two-op chain, this path does not materialize the intermediate
BF16 activation because FLUX.2 feeds it directly into an FP8 projection.
"""
batch, seq_len, hidden = x.shape
batch, seq_len, hidden, stride = _validate_layernorm_modulate(x, scale, shift)
n_rows = batch * seq_len
rows = 2
out = torch.empty_like(x, dtype=fp8_dtype)
stride = _mod_row_stride(scale, batch, hidden)
with torch.cuda.device(x.device):
_layernorm_modulate_kernel[(triton.cdiv(n_rows, rows),)](
out,
@@ -595,19 +605,13 @@ fused_layernorm_modulate = register_custom_op(
)
def can_use_fused_qk_head_layernorm(q: torch.Tensor, k: torch.Tensor) -> bool:
head_dim = q.shape[-1] if q.dim() == 4 else 0
def can_use_fused_qk_head_layernorm(dtype: torch.dtype, head_dim: int) -> bool:
"""Select the BF16 per-head LayerNorm reduction."""
return (
_is_bf16_cuda(q)
and _is_bf16_cuda(k)
and q.device == k.device
and q.dim() == 4
and q.shape == k.shape
and head_dim % 4 == 0
is_cuda()
and dtype is torch.bfloat16
and 0 < head_dim <= 128
and q.numel() > 0
and q.is_contiguous()
and k.is_contiguous()
and head_dim % 4 == 0
)
@@ -627,7 +631,23 @@ def fused_qk_head_layernorm(
) -> tuple[torch.Tensor, torch.Tensor]:
"""Per-head ``nn.LayerNorm(dim_head)`` (no affine) over q and k in one
launch, bit-exact vs the eager aten kernel."""
head_dim = q.shape[-1]
shape = q.shape
if not (
_is_bf16_cuda(q)
and len(shape) == 4
and q.numel() > 0
and 0 < shape[-1] <= 128
and shape[-1] % 4 == 0
and k.dtype is torch.bfloat16
and k.device == q.device
and k.shape == shape
and q.is_contiguous()
and k.is_contiguous()
):
raise RuntimeError(
"QK LayerNorm expects matching contiguous BF16 CUDA [B, S, H, D], D divisible by 4 and <= 128"
)
head_dim = shape[-1]
n_rows = q.numel() // head_dim
# Architecture sweeps at the production GLM shape select 32 rows / 1 warp
# on B300 (SM103) and 8 rows / 4 warps on RTX 5090 (SM120). Preserve the
@@ -1,6 +1,5 @@
from __future__ import annotations
import logging
from typing import TYPE_CHECKING
import torch
@@ -19,9 +18,6 @@ _SUPPORTED_DTYPES = (torch.float16, torch.bfloat16, torch.float32)
_BIT_EXACT_DTYPES = (torch.float16, torch.bfloat16)
_TRANSPOSE_TILE = 32
_MAX_GRID_DIM = 65535
_FAILED_RUNTIME_KEYS: set[tuple[int | None, torch.dtype]] = set()
logger = logging.getLogger(__name__)
@cache_once
@@ -38,10 +34,6 @@ def _jit_residual_gate_add_module(dtype: torch.dtype) -> Module:
"residual_gate_add",
f"residual_gate_add::ResidualGateAddKernel<{args}>::run",
),
(
"residual_gate_add_transposed",
f"residual_gate_add::ResidualGateAddKernel<{args}>::run_transposed",
),
],
)
@@ -67,9 +59,6 @@ def _residual_gate_add_cuda_impl(
device=residual.device,
)
module = _jit_residual_gate_add_module(residual.dtype)
if _is_transposed_dense_residual(residual, update, gate):
module.residual_gate_add_transposed(out, residual, update, gate)
return out
gate_mode = _gate_mode(residual, gate)
module.residual_gate_add(
out.view(-1),
@@ -96,120 +85,6 @@ def _round16_f32(x, IS_BF16: tl.constexpr):
return bits.to(tl.float16, bitcast=True).to(tl.float32)
@triton.jit
def _store16(out_ptr, offs, value_f32, mask, IS_BF16: tl.constexpr):
if IS_BF16:
bits = tl.inline_asm_elementwise(
"cvt.rn.bf16.f32 $0, $1;",
"=h,r",
[value_f32],
dtype=tl.int16,
is_pure=True,
pack=1,
)
tl.store(out_ptr + offs, bits.to(tl.bfloat16, bitcast=True), mask=mask)
else:
bits = tl.inline_asm_elementwise(
"cvt.rn.f16.f32 $0, $1;",
"=h,r",
[value_f32],
dtype=tl.int16,
is_pure=True,
pack=1,
)
tl.store(out_ptr + offs, bits.to(tl.float16, bitcast=True), mask=mask)
@triton.jit
def _rga_flat(
out,
res,
upd,
gate,
numel,
hid,
MODE: tl.constexpr,
IS_BF16: tl.constexpr,
IS_16: tl.constexpr,
BLK: tl.constexpr,
):
pid = tl.program_id(0).to(tl.int64)
offs = pid * BLK + tl.arange(0, BLK).to(tl.int64)
mask = offs < numel
rv = tl.load(res + offs, mask=mask)
uv = tl.load(upd + offs, mask=mask)
if MODE == 1:
gv = tl.load(gate + (offs % hid), mask=mask)
elif MODE == 2:
gv = tl.load(gate + (offs // hid), mask=mask)
else:
gv = tl.load(gate + offs, mask=mask)
if IS_16:
p32 = uv.to(tl.float32) * gv.to(tl.float32)
pf = _round16_f32(p32, IS_BF16)
o32 = rv.to(tl.float32) + pf
_store16(out, offs, o32, mask, IS_BF16)
else:
p32 = uv * gv
o32 = rv + p32
tl.store(out + offs, o32, mask=mask)
@triton.jit
def _rga_row(
out,
res,
upd,
gate,
hid: tl.constexpr,
IS_BF16: tl.constexpr,
EVG: tl.constexpr,
CG: tl.constexpr,
BLK: tl.constexpr,
):
row = tl.program_id(0).to(tl.int64)
offs = tl.arange(0, BLK)
mask = offs < hid
if EVG:
gv = tl.load(gate + offs, mask=mask, eviction_policy="evict_last")
else:
gv = tl.load(gate + offs, mask=mask)
base = row * hid
if CG:
rv = tl.load(res + base + offs, mask=mask, cache_modifier=".cg")
uv = tl.load(upd + base + offs, mask=mask, cache_modifier=".cg")
else:
rv = tl.load(res + base + offs, mask=mask, eviction_policy="evict_first")
uv = tl.load(upd + base + offs, mask=mask, eviction_policy="evict_first")
p32 = uv.to(tl.float32) * gv.to(tl.float32)
pf = _round16_f32(p32, IS_BF16)
o32 = rv.to(tl.float32) + pf
_store16(out, base + offs, o32, mask, IS_BF16)
@triton.jit
def _rga_flat_m1(
out,
res,
upd,
gate,
numel,
hid: tl.constexpr,
IS_BF16: tl.constexpr,
BLK: tl.constexpr,
):
pid = tl.program_id(0).to(tl.int64)
offs = pid * BLK + tl.arange(0, BLK).to(tl.int64)
mask = offs < numel
rv = tl.load(res + offs, mask=mask, eviction_policy="evict_first")
uv = tl.load(upd + offs, mask=mask, eviction_policy="evict_first")
gv = tl.load(gate + (offs % hid), mask=mask, eviction_policy="evict_last")
p32 = uv.to(tl.float32) * gv.to(tl.float32)
pf = _round16_f32(p32, IS_BF16)
o32 = rv.to(tl.float32) + pf
_store16(out, offs, o32, mask, IS_BF16)
@triton.jit
def _rga_transposed(
out,
@@ -268,146 +143,30 @@ def _rga_transposed(
tl.store(out + base + roffs, rv + uv * gv[:, None], mask=m2)
def _residual_gate_add_triton(residual, update, gate):
# Both callers validate the inputs first; repeating the predicate here cost
# ~2.7us of host time on every call.
def _residual_gate_add_transposed(residual, update, gate):
out = torch.empty_strided(
residual.shape, residual.stride(), dtype=residual.dtype, device=residual.device
)
is_bf16 = residual.dtype == torch.bfloat16
dtype16 = residual.dtype in (torch.float16, torch.bfloat16)
if _is_transposed_dense_residual(residual, update, gate):
tokens = residual.shape[1]
hid = residual.shape[2]
batch = residual.shape[0]
if tokens * hid * batch <= 65536:
tile, warps = 16, 4
elif tokens * hid * batch >= 1 << 20:
tile, warps = 64, 8
else:
tile, warps = 32, 8
grid = (triton.cdiv(tokens, tile), triton.cdiv(hid, tile), batch)
_rga_transposed[grid](
out,
residual,
update,
gate,
tokens,
hid,
is_bf16,
dtype16,
TILE=tile,
num_warps=warps,
)
return out
if not dtype16:
# fp32: exact fp32 math, plain Triton elementwise
numel = residual.numel()
hid = residual.shape[-1]
if gate.shape == residual.shape:
mode = 0
elif _is_row_broadcast_gate(residual, gate):
mode = 1
else:
mode = 2
if numel <= 8192:
_rga_flat[(triton.cdiv(numel, 256),)](
out,
residual,
update,
gate,
numel,
hid,
mode,
False,
False,
BLK=256,
num_warps=4,
)
else:
_rga_flat[(triton.cdiv(numel, 1024),)](
out,
residual,
update,
gate,
numel,
hid,
mode,
False,
False,
BLK=1024,
num_warps=4,
)
return out
hid = residual.shape[-1]
numel = residual.numel()
if _is_row_broadcast_gate(residual, gate):
rows = numel // hid
if hid >= 4096 or (rows <= 512 and numel >= 1 << 20):
_rga_flat_m1[(triton.cdiv(numel, 1024),)](
out, residual, update, gate, numel, hid, is_bf16, BLK=1024, num_warps=4
)
elif hid <= 16384:
blk = triton.next_power_of_2(hid)
_rga_row[(rows,)](
out,
residual,
update,
gate,
hid,
is_bf16,
hid < 4608,
hid >= 4096,
BLK=blk,
num_warps=4 if hid >= 4608 else 8,
)
else:
_rga_flat[(triton.cdiv(numel, 1024),)](
out,
residual,
update,
gate,
numel,
hid,
1,
is_bf16,
True,
BLK=1024,
num_warps=4,
)
return out
mode = 0 if gate.shape == residual.shape else 2
if numel <= 8192:
_rga_flat[(triton.cdiv(numel, 256),)](
out,
residual,
update,
gate,
numel,
hid,
mode,
is_bf16,
True,
BLK=256,
num_warps=4,
)
batch, tokens, hidden = residual.shape
if residual.numel() <= 65536:
tile, warps = 16, 4
elif residual.numel() >= 1 << 20:
tile, warps = 64, 8
else:
_rga_flat[(triton.cdiv(numel, 1024),)](
out,
residual,
update,
gate,
numel,
hid,
mode,
is_bf16,
True,
BLK=1024,
num_warps=4,
)
tile, warps = 32, 8
grid = (triton.cdiv(tokens, tile), triton.cdiv(hidden, tile), batch)
_rga_transposed[grid](
out,
residual,
update,
gate,
tokens,
hidden,
residual.dtype == torch.bfloat16,
residual.dtype in (torch.float16, torch.bfloat16),
TILE=tile,
num_warps=warps,
)
return out
@@ -424,7 +183,7 @@ def _residual_gate_add_custom_op(
# benchmarked on. For the contiguous layouts every other diffusion model
# uses, the existing JIT CUDA kernel is faster at every measured shape.
if _is_transposed_dense_residual(residual, update, gate):
return _residual_gate_add_triton(residual, update, gate)
return _residual_gate_add_transposed(residual, update, gate)
return _residual_gate_add_cuda_impl(residual, update, gate)
@@ -472,7 +231,8 @@ def can_use_residual_gate_add_cuda(
residual: torch.Tensor, update: torch.Tensor, gate: torch.Tensor
) -> bool:
return (
residual.dtype in _SUPPORTED_DTYPES
torch.version.hip is None
and residual.dtype in _SUPPORTED_DTYPES
and residual.dtype == update.dtype
and residual.dtype == gate.dtype
and residual.is_cuda
@@ -506,29 +266,11 @@ def residual_gate_add_cuda(
def residual_gate_add(
residual: torch.Tensor, update: torch.Tensor, gate: torch.Tensor
) -> torch.Tensor:
"""Use the bit-exact CUDA fast path when supported, otherwise eager.
Runtime build failures are cached per device and dtype so every diffusion
model shares one fallback policy instead of maintaining model-local flags.
"""
runtime_key = (residual.device.index, residual.dtype)
if (
residual.dtype in _BIT_EXACT_DTYPES
and runtime_key not in _FAILED_RUNTIME_KEYS
and can_use_residual_gate_add_cuda(residual, update, gate)
"""Use the bit-exact fast path for supported layouts, otherwise eager."""
if residual.dtype in _BIT_EXACT_DTYPES and can_use_residual_gate_add_cuda(
residual, update, gate
):
try:
return _residual_gate_add_custom_op(residual, update, gate)
except Exception as exc:
if torch.compiler.is_compiling():
raise
_FAILED_RUNTIME_KEYS.add(runtime_key)
logger.warning(
"Disabling diffusion residual-gate CUDA fast path on %s/%s: %s",
residual.device,
residual.dtype,
exc,
)
return _residual_gate_add_custom_op(residual, update, gate)
return residual + update * gate
+42 -19
View File
@@ -80,18 +80,41 @@ GEMM in FP32. The default path still promotes Q/K/V before both GEMMs.
## Entry-point protocol
Every public kernel is a **predicate + kernel** pair:
Keep backend selection separate from input validation.
```python
if can_use_<op>(...):
out = <op>(...)
else:
out = <reference chain>
```
- Select a backend using properties that distinguish supported implementations:
device family, dtype, layout, alignment, architecture, and numerical contract.
A shape restriction belongs here only when it is a real specialization, such
as a fixed head dimension or an unsupported broadcast mode.
Use the input's device when selecting CPU versus CUDA: a CUDA-capable
process can still receive CPU tensors. A CPU fallback must not disable a
process-wide GPU fusion gate.
- Validate the selected implementation's inputs before launching. For JIT CUDA,
use `TensorMatcher` and `CHECK_HOST` in the C++ launcher; keep dtype checks in
the cached module factory. For Triton, validate in the Python launch wrapper.
Mismatched shapes or devices are errors, not reasons to silently try another
backend. Preserve tensor dimensions until they have been validated.
- Add a `can_use_*` predicate only when a caller needs a backend choice. A direct
kernel entry point does not need a second copy of the same predicate. Keep
fallback selection in one wrapper when several callers share it.
- Let errors from a selected implementation propagate. Do not catch every
exception and permanently disable a kernel for later requests. First-use
numerical verification in `sites/` is a separate policy: retain its reference
comparison when exactness depends on a library's reduction or rounding order.
The kernel raises on an unsupported input. It does not return `None` — a
silent `None` is too easy to forget to check, and the failure mode is a
wrong-looking image rather than an exception.
For example, Helios selects JIT CUDA RoPE on NVIDIA GPUs unless it uses TP
RMSNorm. The launcher validates the original batched Q/K and frequencies,
including shape relationships, dtype, device, layout and pair alignment.
There is no Python `can_use_helios_qk_rope` scan before the call.
A numerical specialization query should take configuration values, not a list
of tensors. For example, `can_use_fused_rmsnorm_scale_shift(dtype, hidden)`
selects BF16 with width 2048 or 4096 because those are the FlashInfer reduction
trees the kernel reproduces. Its launcher validates weight, scale and shift.
Existing `try_*` entry points return `None` for an unsupported specialization.
Keep that convention explicit at their call sites; do not add it to direct
kernel entry points.
## Selection matrix
@@ -127,13 +150,12 @@ Several norms look interchangeable and are not. Start here.
| Entry point | Backend | Contract | Applies to |
|---|---|---|---|
| `residual_gate_add` | KDA (JIT CUDA) | bit-exact `residual + update * gate` | contiguous tensors, or a transposed-dense `[B, tokens, hidden]` residual/output with contiguous update and row-broadcast gate (SANA-Video) |
| `residual_gate_add` | JIT CUDA (contiguous), Triton (transposed) | bit-exact `residual + update * gate` | contiguous tensors, or a transposed-dense `[B, tokens, hidden]` residual/output with contiguous update and row-broadcast gate (SANA-Video) |
The transposed-dense path uses a shared-memory tile to read the update in
logical row-major order while keeping residual reads and output writes
coalesced in their `[B, hidden, tokens]` backing layout. Do not insert a
`.contiguous()` merely to reach the ordinary path; that restores an entire
tensor copy per residual site.
The transposed-dense path tiles along the physical stride-1 token dimension
for coalesced residual reads and output writes. The contiguous layouts use
JIT CUDA. Do not insert a `.contiguous()` to reach that path: it adds a full
tensor copy at each residual site.
### RoPE / QK-norm
@@ -146,8 +168,8 @@ tensor copy per residual site.
| `try_fused_qwen_qkv_epilogue` | JIT CUDA | bit-exact vs the selected BF16 chain | Qwen-Image QK RMSNorm + RoPE + joint QKV writes; SM90+ |
| `fused_rope_rotate_half_bitexact` | Triton | bit-exact (elementwise only) |
| `fused_complex_rope` | Triton | preserves CUDA complex64 multiply rounding for contiguous BSHD inputs; Qwen-Image 2.1 verifies its first call against eager |
| `rmsnorm_preserve_reduction` | Triton + aten | preserves the FP32 mean reduction and cast-before-weight rounding; fuses only pointwise work for contiguous FP16/BF16 inputs; Qwen-Image 2.1 verifies its first call |
| `qknorm_complex_rope_cuda` / `qknorm_complex_rope_pack_` | JIT CUDA | bit-exact vs `RMSNorm(cast_x_before_out_mul=True)` + complex64 RoPE for head_dim 128; the pack variant normalizes Q and K in place behind a prefix, copies prefix K/V (and optionally V) in the same launch, and accepts token-strided views of a packed QKV buffer; Qwen-Image 2.1 verifies its first call |
| `rmsnorm_preserve_reduction` | Triton + aten | preserves the FP32 mean reduction and cast-before-weight rounding; fuses only pointwise work for FP16/BF16 inputs and materializes strided inputs as contiguous; Qwen-Image 2.1 verifies its first call |
| `qknorm_complex_rope_cuda` / `qknorm_complex_rope_pack_` | JIT CUDA | bit-exact vs `RMSNorm(cast_x_before_out_mul=True)` + complex64 RoPE for head_dim 128; both accept token-strided views of a packed QKV buffer; the pack variant normalizes Q and K in place behind a prefix and copies prefix K/V (and optionally V) in the same launch; Qwen-Image 2.1 verifies its first call |
| `fused_interleaved_rope_fp64` | JIT CUDA | bit-exact vs paired SANA-Video fp64 RoPE |
| `fused_inplace_helios_qk_rope` | JIT CUDA | bit-exact paired in-place RoPE for Helios' transposed frequency layout |
| `ltx2_qknorm_split_rope_cuda` | KDA (JIT CUDA) | close; **validated on B200** |
@@ -218,7 +240,8 @@ inspecting model modules is its whole job.
with its source revision and any JIT CUDA source files.
2. Export it from `__init__.py` (`_EXPORTS`) and register a `KernelSpec`
(`_SPECS`).
3. Give it a `can_use_*` predicate; raise, don't return `None`.
3. Keep selection and validation separate as described above. Reuse an existing
wrapper when the fallback policy is shared.
4. State the numerical contract in the module docstring, including which
shapes it was verified on.
5. If it is not bit-exact, gate it through `sites/`. It must mount for both
@@ -564,7 +564,6 @@ for _op, _backend, _target, _caps, _description in _SPECS:
_EXPORTS: dict[str, str] = {
"can_use_fp8_rowwise": "quantization.fp8_rowwise_triton",
"fp8_rowwise": "quantization.fp8_rowwise_triton",
"can_use_fused_gelu_tanh_cat": "activation.gelu_tanh_cat_jit",
"fused_gelu_tanh_cat": "activation.gelu_tanh_cat_jit",
"load_extension_with_recovery": "sglang.srt.utils.cpp_extension_loader",
# Normalization: RMSNorm / LayerNorm / GroupNorm and their fused epilogues
@@ -598,10 +597,8 @@ _EXPORTS: dict[str, str] = {
"triton_one_pass_rms_norm": "norm.rmsnorm_onepass_triton",
"can_use_flux2_strided_qknorm_rope": "rope.flux2_qknorm_rope_triton",
"flux2_strided_qknorm_rope": "rope.flux2_qknorm_rope_triton",
"can_use_rmsnorm_preserve_reduction": "norm.rmsnorm_preserve_reduction",
"rmsnorm_preserve_reduction": "norm.rmsnorm_preserve_reduction",
"can_use_fused_rmsnorm_scale_shift": "norm.rmsnorm_scale_shift_bitexact",
"can_use_fused_scale_residual_rmsnorm_scale_shift": "norm.rmsnorm_scale_shift_bitexact",
"fused_rmsnorm_scale_shift_bitexact": "norm.rmsnorm_scale_shift_bitexact",
"fused_scale_residual_rmsnorm_scale_shift_bitexact": "norm.rmsnorm_scale_shift_bitexact",
"fused_norm_scale_shift": "norm.scale_residual_norm_cutedsl",
@@ -636,43 +633,32 @@ _EXPORTS: dict[str, str] = {
"try_fused_scaled_residual_bf16": "modulate.scale_shift_triton",
"try_fused_scaled_residual_add_exact": "modulate.scale_shift_triton",
"timestep_embedding": "modulate.timestep_embedding_jit",
"can_use_fused_temb_table_slices": "modulate.wan_temb_table_slices_triton",
"fused_temb_table_slices": "modulate.wan_temb_table_slices_triton",
# Rotary embeddings and the QK-norm chains fused around them
"try_fused_flux2_qkv_epilogue": "sglang.kernels.kda_kernels.flux2_qkv_epilogue_jit",
"hunyuan_qkv_rope_pack": "rope.hunyuan_qkv_pack_triton",
"can_use_joint_qkv_cat": "layout.joint_qkv_cat_triton",
"joint_qkv_cat": "layout.joint_qkv_cat_triton",
"can_use_ltx2_qknorm_split_rope_cuda": "sglang.kernels.kda_kernels.ltx2_qknorm_split_rope_jit",
"ltx2_qknorm_split_rope_cuda": "sglang.kernels.kda_kernels.ltx2_qknorm_split_rope_jit",
"apply_ltx2_split_rotary_emb": "rope.ltx2_rotary_triton",
"can_use_ltx25_decoder_rope": "rope.ltx25_decoder_rope_jit",
"fused_ltx25_decoder_rope": "rope.ltx25_decoder_rope_jit",
"can_use_fused_inplace_qknorm_rope": "rope.qknorm_rope_jit",
"fused_inplace_qknorm_rope": "rope.qknorm_rope_jit",
"fused_qknorm_rope_pack_kv": "rope.qknorm_rope_jit",
"try_fused_qwen_qkv_epilogue": "rope.qwen_qkv_epilogue_jit",
"can_use_fused_rope_rotate_half": "rope.rope_rotate_half_bitexact",
"fused_rope_rotate_half_bitexact": "rope.rope_rotate_half_bitexact",
"can_use_interleaved_rope_fp64": "rope.interleaved_rope_fp64_jit",
"fused_interleaved_rope_fp64": "rope.interleaved_rope_fp64_jit",
"can_use_helios_qk_rope": "rope.helios_qk_rope_jit",
"fused_inplace_helios_qk_rope": "rope.helios_qk_rope_jit",
"apply_rotary_embedding": "rope.rotary_triton",
"can_use_fused_complex_rope": "rope.complex_rope_triton",
"fused_complex_rope": "rope.complex_rope_triton",
"can_use_qknorm_complex_rope_cuda": "rope.qknorm_complex_rope_jit",
"can_use_qknorm_complex_rope_pack": "rope.qknorm_complex_rope_jit",
"qknorm_complex_rope_cuda": "rope.qknorm_complex_rope_jit",
"qknorm_complex_rope_pack_": "rope.qknorm_complex_rope_jit",
# Tensor layout transformations fused with downstream quantization
"try_flux2_token_cat_fp8": "sglang.kernels.kda_kernels.flux2_token_cat_fp8_triton",
# Activation-function fusions
"can_use_fused_bias_glu": "activation.sana_conv_post_triton",
"can_use_fused_bias_silu": "activation.sana_conv_post_triton",
"fused_bias_glu": "activation.sana_conv_post_triton",
"fused_bias_silu": "activation.sana_conv_post_triton",
"can_use_fused_silu_mul": "activation.silu_mul_bitexact",
"fused_packed_silu_mul_bitexact": "activation.silu_mul_bitexact",
"fused_silu_mul_bitexact": "activation.silu_mul_bitexact",
# Diffusion attention kernels
@@ -757,7 +743,6 @@ _EXPORTS: dict[str, str] = {
"mount_hunyuan_qknorm": "sites.hunyuan_qknorm_site",
"try_hunyuan_qknorm": "sites.hunyuan_qknorm_site",
"unmount_hunyuan_qknorm": "sites.hunyuan_qknorm_site",
"can_use_ltx2_rms_norm_modulate": "sites.ltx2_rmsnorm_modulate_site",
"fused_ltx2_rms_norm_modulate": "sites.ltx2_rmsnorm_modulate_site",
"ltx2_rms_norm_modulate_active": "sites.ltx2_rmsnorm_modulate_site",
"mark_ltx2_rms_norm_modulate_site": "sites.ltx2_rmsnorm_modulate_site",
@@ -24,34 +24,9 @@ def _module():
)
def can_use_fused_gelu_tanh_cat(attn: torch.Tensor, mlp: torch.Tensor) -> bool:
return (
not torch.is_grad_enabled()
and not torch.compiler.is_compiling()
and torch.version.hip is None
and attn.is_cuda
and attn.dtype == mlp.dtype == torch.bfloat16
and attn.device == mlp.device
and attn.ndim >= 2
and attn.shape[:-1] == mlp.shape[:-1]
and attn.shape[-1] > 0
and mlp.shape[-1] > 0
and attn.shape[-1] % 8 == mlp.shape[-1] % 8 == 0
and attn.is_contiguous()
and mlp.is_contiguous()
and attn.numel() > 0
and (attn.numel() + mlp.numel()) // 8 <= 2**31 - 1
and attn.data_ptr() % 16 == mlp.data_ptr() % 16 == 0
)
@register_custom_op(mutates_args=["output"])
def _gelu_tanh_cat(attn: torch.Tensor, mlp: torch.Tensor, output: torch.Tensor) -> None:
_module().gelu_tanh_cat(
attn.view(-1, attn.shape[-1]),
mlp.view(-1, mlp.shape[-1]),
output.view(-1, output.shape[-1]),
)
_module().gelu_tanh_cat(attn, mlp, output)
def fused_gelu_tanh_cat(attn: torch.Tensor, mlp: torch.Tensor) -> torch.Tensor:
@@ -72,31 +72,30 @@ def _bias_glu_kernel(
tl.store(out_ptr + offsets, hidden * gate, mask=mask)
def _is_dense_bf16(x: torch.Tensor) -> bool:
return (
def _validate_conv_post(x: torch.Tensor, bias: torch.Tensor | None) -> None:
if not (
x.is_cuda
and x.dtype is torch.bfloat16
and x.dim() == 4
and x.ndim == 4
and x.numel() > 0
and (x.is_contiguous() or x.is_contiguous(memory_format=torch.channels_last))
)
def can_use_fused_bias_silu(x: torch.Tensor, bias: torch.Tensor) -> bool:
return (
_is_dense_bf16(x)
and bias.is_cuda
and bias.dtype is x.dtype
):
raise RuntimeError(
"Sana conv post-processing expects a dense BF16 CUDA NCHW tensor"
)
if bias is not None and not (
bias.dtype == x.dtype
and bias.device == x.device
and bias.dim() == 1
and bias.shape[0] == x.shape[1]
and bias.shape == (x.shape[1],)
and bias.is_contiguous()
)
):
raise RuntimeError(
"bias must be a contiguous channel vector matching x's dtype and device"
)
def fused_bias_silu(x: torch.Tensor, bias: torch.Tensor) -> torch.Tensor:
if not can_use_fused_bias_silu(x, bias):
raise RuntimeError("unsupported input for Sana fused bias-SiLU")
_validate_conv_post(x, bias)
out = torch.empty_like(x, memory_format=torch.preserve_format)
with torch.cuda.device(x.device):
_bias_silu_kernel[(triton.cdiv(x.numel(), 1024),)](
@@ -111,32 +110,15 @@ def fused_bias_silu(x: torch.Tensor, bias: torch.Tensor) -> torch.Tensor:
return out
def can_use_fused_bias_glu(x: torch.Tensor, bias: torch.Tensor | None) -> bool:
return (
_is_dense_bf16(x)
and x.shape[1] % 2 == 0
and (
bias is None
or (
bias.is_cuda
and bias.dtype is x.dtype
and bias.device == x.device
and bias.dim() == 1
and bias.shape[0] == x.shape[1]
and bias.is_contiguous()
)
)
)
def fused_bias_glu(x: torch.Tensor, bias: torch.Tensor | None) -> torch.Tensor:
"""Apply optional bias then ``hidden * silu(gate)`` along the channel axis.
Pass no bias for an already biased native depthwise-convolution output:
splitting that convolution's bias can change its accumulation rounding.
"""
if not can_use_fused_bias_glu(x, bias):
raise RuntimeError("unsupported input for Sana fused bias-GLU")
_validate_conv_post(x, bias)
if x.shape[1] % 2:
raise RuntimeError("Sana GLU requires an even channel count")
batch, double_channels, height, width = x.shape
channels = double_channels // 2
channels_last = x.is_contiguous(memory_format=torch.channels_last)
@@ -161,8 +143,6 @@ def fused_bias_glu(x: torch.Tensor, bias: torch.Tensor | None) -> torch.Tensor:
__all__ = [
"can_use_fused_bias_glu",
"can_use_fused_bias_silu",
"fused_bias_glu",
"fused_bias_silu",
]
@@ -60,20 +60,6 @@ def _packed_silu_mul_kernel(
tl.store(out_ptr + row * D + cols, s * b, mask=mask)
def can_use_fused_silu_mul(a: torch.Tensor, b: torch.Tensor) -> bool:
return (
a.dtype is torch.bfloat16
and b.dtype is torch.bfloat16
and a.is_cuda
and b.is_cuda
and a.device == b.device
and a.shape == b.shape
and a.is_contiguous()
and b.is_contiguous()
and a.numel() > 0
)
def _fake_silu_mul(a: torch.Tensor, b: torch.Tensor) -> torch.Tensor:
return torch.empty_like(a)
@@ -85,6 +71,18 @@ def _fake_silu_mul(a: torch.Tensor, b: torch.Tensor) -> torch.Tensor:
)
def fused_silu_mul_bitexact(a: torch.Tensor, b: torch.Tensor) -> torch.Tensor:
"""``silu(a) * b``, bit-exact vs the eager two-kernel chain."""
if not (
a.dtype is torch.bfloat16
and b.dtype is torch.bfloat16
and a.is_cuda
and b.is_cuda
and a.device == b.device
and a.shape == b.shape
and a.is_contiguous()
and b.is_contiguous()
and a.numel() > 0
):
raise RuntimeError("invalid input for fused_silu_mul_bitexact")
out = torch.empty_like(a)
numel = a.numel()
with torch.cuda.device(a.device):
@@ -65,34 +65,22 @@ def vdn_delta_factors(
transition = torch.empty_like(A)
injection = torch.empty_like(B)
module = _jit_vdn_delta_factors_module()
module.vdn_delta_factors(
transition.view(-1, HEAD_DIM, HEAD_DIM),
injection.view(-1, HEAD_DIM, HEAD_DIM),
A.view(-1, HEAD_DIM, HEAD_DIM),
B.view(-1, HEAD_DIM, HEAD_DIM),
alpha.view(-1, HEAD_DIM),
)
module.vdn_delta_factors(transition, injection, A, B, alpha)
return transition, injection
@torch.compiler.assume_constant_result
@cache_once
def can_use_vdn_delta_factors(
A: torch.Tensor, B: torch.Tensor, alpha: torch.Tensor
device: torch.device, dtype: torch.dtype, head_dim: int
) -> bool:
"""The register inverse is specialized to FP32, 128-wide heads on SM80+."""
return (
A.is_cuda
and A.dtype is torch.float32
and B.dtype is torch.float32
and alpha.dtype is torch.float32
and A.device == B.device == alpha.device
and A.dim() >= 2
and A.shape[-1] == HEAD_DIM
and A.shape[-2] == HEAD_DIM
and B.shape == A.shape
and alpha.shape == A.shape[:-1]
and A.is_contiguous()
and B.is_contiguous()
and alpha.is_contiguous()
and torch.cuda.get_device_capability(A.device)[0] >= 8
device.type == "cuda"
and not torch.version.hip
and dtype is torch.float32
and head_dim == HEAD_DIM
and torch.cuda.get_device_capability(device)[0] >= 8
)
@@ -55,35 +55,6 @@ def _joint_qkv_cat_kernel(
tl.store(OUT + (component * BATCH * sequence + row) * HIDDEN + cols, value, mask)
def can_use_joint_qkv_cat(*inputs: torch.Tensor) -> bool:
if len(inputs) != 6 or torch.compiler.is_compiling() or torch.version.hip:
return False
first = inputs[0]
if first.ndim != 4 or not first.is_cuda:
return False
if first.dtype not in (torch.float16, torch.bfloat16):
return False
batch, tokens, heads, dim = first.shape
if min(batch, tokens, heads, dim) <= 0 or heads * dim > 8192:
return False
for index, value in enumerate(inputs):
expected = first.shape if index < 3 else inputs[3].shape
if (
value.ndim != 4
or value.shape != expected
or value.shape[0] != batch
or value.shape[2:] != (heads, dim)
or value.shape[1] <= 0
or value.dtype != first.dtype
or value.device != first.device
or value.requires_grad
or value.stride(-1) != 1
or value.stride(-2) != dim
):
return False
return True
def joint_qkv_cat(
img_q: torch.Tensor,
img_k: torch.Tensor,
@@ -98,7 +69,33 @@ def joint_qkv_cat(
The three outputs occupy disjoint, contiguous regions of one allocation.
"""
inputs = (img_q, img_k, img_v, txt_q, txt_k, txt_v)
assert can_use_joint_qkv_cat(*inputs)
if not (
img_q.is_cuda
and img_q.dtype in (torch.float16, torch.bfloat16)
and img_q.ndim == 4
and min(img_q.shape) > 0
and img_q.shape[-2] * img_q.shape[-1] <= 8192
):
raise RuntimeError(
"joint QKV expects FP16/BF16 CUDA [B, S, H, D], 0 < H * D <= 8192"
)
for index, value in enumerate(inputs):
expected = img_q.shape if index < 3 else txt_q.shape
if not (
value.ndim == 4
and value.shape == expected
and value.shape[0] == img_q.shape[0]
and value.shape[2:] == img_q.shape[2:]
and value.shape[1] > 0
and value.dtype == img_q.dtype
and value.device == img_q.device
and not value.requires_grad
and value.stride(-1) == 1
and value.stride(-2) == img_q.shape[-1]
):
raise RuntimeError(
"joint QKV inputs must share batch/head shape, dtype/device and have packed head rows without gradients"
)
batch, image_tokens, heads, dim = img_q.shape
text_tokens = txt_q.shape[1]
output = torch.empty(
@@ -42,7 +42,7 @@ def _fake_merge_heads(x: torch.Tensor) -> torch.Tensor:
mutates_args=[],
fake_impl=_fake_merge_heads,
)
def _usp_merge_heads_custom_op(x: torch.Tensor) -> torch.Tensor:
def _usp_merge_heads_cuda(x: torch.Tensor) -> torch.Tensor:
world, seq, batch, h_local, head_dim = x.shape
out = x.new_empty((batch, seq, world, h_local, head_dim))
module = _jit_usp_relayout_module(x.dtype)
@@ -52,27 +52,14 @@ def _usp_merge_heads_custom_op(x: torch.Tensor) -> torch.Tensor:
def can_use_usp_merge_heads(x: torch.Tensor) -> bool:
return (
isinstance(x, torch.Tensor)
and torch.version.hip is None
torch.version.hip is None
and x.is_cuda
and x.dtype in _SUPPORTED_DTYPES
and x.dim() == 5
and x.numel() > 0
and x.is_contiguous()
)
def _usp_merge_heads_cuda(x: torch.Tensor) -> torch.Tensor:
"""[W, S, B, h_local, D] -> [B, S, W, h_local, D] contiguous.
Bit-exact single-pass replacement for
``x.permute(2, 1, 0, 3, 4).contiguous()`` on the Ulysses output path.
"""
if not can_use_usp_merge_heads(x):
raise RuntimeError("unsupported input for usp_merge_heads CUDA")
return _usp_merge_heads_custom_op(x)
def usp_merge_heads(x: torch.Tensor) -> torch.Tensor:
"""Merge Ulysses output heads with an exact eager fallback.
@@ -7,7 +7,6 @@ bit-exact vs the eager chain (``torch.equal``) and needs no quality gate.
from __future__ import annotations
import logging
from typing import TYPE_CHECKING
import torch
@@ -21,9 +20,6 @@ if TYPE_CHECKING:
_SUPPORTED_DTYPES = (torch.float16, torch.bfloat16)
_ALIGN_BYTES = 16
_FAILED_RUNTIME_KEYS: set[tuple[int | None, torch.dtype]] = set()
logger = logging.getLogger(__name__)
@cache_once
@@ -55,9 +51,10 @@ def _fake_impl(
mutates_args=[],
fake_impl=_fake_impl,
)
def _modulate_scale_shift_custom_op(
def modulate_scale_shift_cuda(
x: torch.Tensor, scale: torch.Tensor, shift: torch.Tensor
) -> torch.Tensor:
"""Fused ``x * (1 + scale[:, None]) + shift[:, None]``; validated by CUDA."""
out = torch.empty_like(x)
module = _jit_modulate_scale_shift_module(x.dtype)
module.modulate_scale_shift(out, x, scale, shift)
@@ -72,7 +69,8 @@ def can_use_modulate_scale_shift_cuda(
x: torch.Tensor, scale: torch.Tensor, shift: torch.Tensor
) -> bool:
if (
x.dtype not in _SUPPORTED_DTYPES
torch.version.hip is not None
or x.dtype not in _SUPPORTED_DTYPES
or scale.dtype != x.dtype
or shift.dtype != x.dtype
or not (x.is_cuda and scale.is_cuda and shift.is_cuda)
@@ -91,35 +89,12 @@ def can_use_modulate_scale_shift_cuda(
)
def modulate_scale_shift_cuda(
x: torch.Tensor, scale: torch.Tensor, shift: torch.Tensor
) -> torch.Tensor:
"""Fused ``x * (1 + scale[:, None]) + shift[:, None]`` (bit-exact vs eager)."""
if not can_use_modulate_scale_shift_cuda(x, scale, shift):
raise RuntimeError("unsupported input for modulate_scale_shift CUDA")
return _modulate_scale_shift_custom_op(x, scale, shift)
def modulate_scale_shift(
x: torch.Tensor, scale: torch.Tensor, shift: torch.Tensor
) -> torch.Tensor:
"""Use the bit-exact CUDA fast path when supported, otherwise eager."""
runtime_key = (x.device.index, x.dtype)
if runtime_key not in _FAILED_RUNTIME_KEYS and can_use_modulate_scale_shift_cuda(
x, scale, shift
):
try:
return modulate_scale_shift_cuda(x, scale, shift)
except Exception as exc:
if torch.compiler.is_compiling():
raise
_FAILED_RUNTIME_KEYS.add(runtime_key)
logger.warning(
"Disabling diffusion modulate CUDA fast path on %s/%s: %s",
x.device,
x.dtype,
exc,
)
if can_use_modulate_scale_shift_cuda(x, scale, shift):
return modulate_scale_shift_cuda(x, scale, shift)
return x * (1 + scale[:, None]) + shift[:, None]
@@ -49,22 +49,6 @@ def _temb_table_slices_kernel(
tl.store(out_ptr + (j * rows + row) * D + cols, w + t, mask=mask)
def can_use_fused_temb_table_slices(table: torch.Tensor, temb: torch.Tensor) -> bool:
return (
temb.is_cuda
and temb.dtype in (torch.bfloat16, torch.float16, torch.float32)
and temb.dim() == 4
and temb.shape[2] == 6
and temb.is_contiguous()
and table.is_cuda
and table.device == temb.device
and table.dtype in (torch.bfloat16, torch.float16, torch.float32)
and table.shape == (1, 6, temb.shape[-1])
and table.is_contiguous()
and temb.numel() > 0
)
def _fake_temb_table_slices(table: torch.Tensor, temb: torch.Tensor) -> torch.Tensor:
batch, seq_len, six, hidden = temb.shape
return temb.new_empty((six, batch, seq_len, hidden), dtype=torch.float32)
@@ -81,6 +65,20 @@ def fused_temb_table_slices(table: torch.Tensor, temb: torch.Tensor) -> torch.Te
``temb`` is ``(B, S, 6, D)``; ``table`` is the block's ``(1, 6, D)`` fp32
adaLN table. ``out[j]`` is the ``j``-th modulation slice, contiguous.
"""
if not (
temb.is_cuda
and temb.dtype in (torch.bfloat16, torch.float16, torch.float32)
and temb.dim() == 4
and temb.shape[2] == 6
and temb.is_contiguous()
and table.is_cuda
and table.device == temb.device
and table.dtype in (torch.bfloat16, torch.float16, torch.float32)
and table.shape == (1, 6, temb.shape[-1])
and table.is_contiguous()
and temb.numel() > 0
):
raise RuntimeError("invalid input for fused_temb_table_slices")
batch, seq_len, _, hidden = temb.shape
out = temb.new_empty((6, batch, seq_len, hidden), dtype=torch.float32)
rows = batch * seq_len
@@ -2,7 +2,7 @@
"""Fuse RMSNorm pointwise work while retaining the native FP32 mean reduction.
Matches ``weight * (x.float() * rsqrt(mean(x.float()**2) + eps)).to(x.dtype)``
for contiguous FP16/BF16 inputs. The FP32 square buffer has the original shape,
for FP16/BF16 inputs; packed projection views are materialized before the launch. The FP32 square buffer has the original shape,
so aten selects the same reduction as the eager chain. Verified at head width
128, including 131072 rows; callers verify their first dispatch before reuse.
"""
@@ -42,23 +42,8 @@ def _rmsnorm_finish_kernel(
tl.store(out_ptr + index, normalized.to(tl.float32) * weight, mask)
def can_use_rmsnorm_preserve_reduction(x: torch.Tensor, weight: torch.Tensor) -> bool:
return (
x.is_cuda
and torch.version.hip is None
and x.dtype in (torch.float16, torch.bfloat16)
and x.ndim >= 2
and x.numel() > 0
and x.is_contiguous()
and weight.device == x.device
and weight.dtype == x.dtype
and weight.shape == (x.shape[-1],)
and weight.is_contiguous()
)
def _fake_rmsnorm(x: torch.Tensor, weight: torch.Tensor, eps: float) -> torch.Tensor:
return torch.empty_like(x)
return torch.empty(x.shape, dtype=x.dtype, device=x.device)
@register_custom_op(
@@ -70,7 +55,19 @@ def rmsnorm_preserve_reduction(
x: torch.Tensor, weight: torch.Tensor, eps: float
) -> torch.Tensor:
"""Preserve aten's mean and cast-before-weight semantics without residuals."""
assert can_use_rmsnorm_preserve_reduction(x, weight)
if not (
x.is_cuda
and torch.version.hip is None
and x.dtype in (torch.float16, torch.bfloat16)
and x.ndim >= 2
and x.numel() > 0
and weight.device == x.device
and weight.dtype == x.dtype
and weight.shape == (x.shape[-1],)
and weight.is_contiguous()
):
raise RuntimeError("invalid input for rmsnorm_preserve_reduction")
x = x.contiguous()
squares = torch.empty_like(x, dtype=torch.float32)
out = torch.empty_like(x)
with torch.cuda.device(x.device):
@@ -155,59 +155,57 @@ def _threads_per_row(hidden: int) -> int | None:
return tpr
def _is_row_broadcast(t: torch.Tensor, x: torch.Tensor) -> bool:
def _is_row_broadcast(t: torch.Tensor, shape: tuple, device: torch.device) -> bool:
return (
t.dtype is torch.bfloat16
and t.is_cuda
and t.device == x.device
and t.shape == (x.shape[0], 1, x.shape[-1])
and t.device == device
and t.shape == shape
and t.is_contiguous()
)
def can_use_fused_rmsnorm_scale_shift(
x: torch.Tensor,
weight: torch.Tensor,
scale: torch.Tensor,
shift: torch.Tensor,
) -> bool:
def can_use_fused_rmsnorm_scale_shift(dtype: torch.dtype, hidden: int) -> bool:
"""Select the BF16 FlashInfer reduction replicated by this kernel."""
return (
# ROCm cannot compile the inline PTX above: LLVM makes the unusable
# `=f` constraint a fatal error that kills the process, so reject
# before the first launch rather than rely on the caller's fallback.
is_cuda() and dtype is torch.bfloat16 and _threads_per_row(hidden) is not None
)
def _validate_rmsnorm_modulate(x, weight, scale, shift) -> tuple[int, int, int, int]:
shape = x.shape
if not (
is_cuda()
and x.dtype is torch.bfloat16
and x.is_cuda
and x.dim() == 3
and x.dtype is torch.bfloat16
and len(shape) == 3
and x.is_contiguous()
and _threads_per_row(x.shape[-1]) is not None
and weight.dtype is torch.bfloat16
and weight.is_cuda
and weight.device == x.device
and weight.shape == (x.shape[-1],)
):
raise RuntimeError(
"RMSNorm modulation expects contiguous BF16 CUDA [B, S, D], D in (2048, 4096)"
)
batch, seq_len, hidden = shape
tpr = _threads_per_row(hidden)
if tpr is None:
raise RuntimeError("RMSNorm modulation requires D in (2048, 4096)")
device = x.device
if not (
weight.dtype is torch.bfloat16
and weight.device == device
and weight.shape == (hidden,)
and weight.is_contiguous()
and _is_row_broadcast(scale, x)
and _is_row_broadcast(shift, x)
)
def can_use_fused_scale_residual_rmsnorm_scale_shift(
residual: torch.Tensor,
update: torch.Tensor,
gate: torch.Tensor,
weight: torch.Tensor,
scale: torch.Tensor,
shift: torch.Tensor,
) -> bool:
return (
can_use_fused_rmsnorm_scale_shift(residual, weight, scale, shift)
and update.dtype is torch.bfloat16
and update.is_cuda
and update.device == residual.device
and update.shape == residual.shape
and update.is_contiguous()
and _is_row_broadcast(gate, residual)
)
):
raise RuntimeError(
"weight must be a contiguous [D] vector matching x's dtype/device"
)
row_shape = (batch, 1, hidden)
if not (
_is_row_broadcast(scale, row_shape, device)
and _is_row_broadcast(shift, row_shape, device)
):
raise RuntimeError(
"scale and shift must be contiguous [B, 1, D] tensors matching x's dtype/device"
)
return batch, seq_len, hidden, tpr
def _fake_norm_scale_shift(
@@ -233,8 +231,7 @@ def fused_rmsnorm_scale_shift_bitexact(
eps: float,
) -> torch.Tensor:
"""``norm(x) * (1 + scale) + shift``, bit-exact vs the eager chain."""
batch, seq_len, hidden = x.shape
tpr = _threads_per_row(hidden)
batch, seq_len, hidden, tpr = _validate_rmsnorm_modulate(x, weight, scale, shift)
out = torch.empty_like(x)
with torch.cuda.device(x.device):
_rmsnorm_scale_shift_kernel[(batch * seq_len,)](
@@ -291,8 +288,19 @@ def fused_scale_residual_rmsnorm_scale_shift_bitexact(
Returns ``(modulated, res)``, both bit-exact vs the eager pair + norm
chain (and therefore vs the ``residual_gate_add_cuda`` fast path).
"""
batch, seq_len, hidden = residual.shape
tpr = _threads_per_row(hidden)
batch, seq_len, hidden, tpr = _validate_rmsnorm_modulate(
residual, weight, scale, shift
)
if not (
update.dtype == residual.dtype
and update.device == residual.device
and update.shape == residual.shape
and update.is_contiguous()
and _is_row_broadcast(gate, (batch, 1, hidden), residual.device)
):
raise RuntimeError(
"update must match residual; gate must be contiguous [B, 1, D] with the same dtype/device"
)
out = torch.empty_like(residual)
res_out = torch.empty_like(residual)
with torch.cuda.device(residual.device):
@@ -50,22 +50,6 @@ def _fuse_real_sin(device: torch.device) -> bool:
return (z[0] * z[1]).imag.item() != 0
def can_use_fused_complex_rope(x: torch.Tensor, rope: torch.Tensor) -> bool:
return (
x.is_cuda
and torch.version.hip is None
and x.dtype in (torch.float16, torch.bfloat16, torch.float32)
and x.ndim == 4
and x.numel() > 0
and x.shape[-1] % 2 == 0
and x.is_contiguous()
and rope.dtype == torch.complex64
and rope.device == x.device
and rope.shape == (x.shape[1], x.shape[-1] // 2)
and rope.is_contiguous()
)
def _fake_complex_rope(x: torch.Tensor, rope: torch.Tensor) -> torch.Tensor:
return torch.empty_like(x)
@@ -77,7 +61,20 @@ def _fake_complex_rope(x: torch.Tensor, rope: torch.Tensor) -> torch.Tensor:
)
def fused_complex_rope(x: torch.Tensor, rope: torch.Tensor) -> torch.Tensor:
"""Rotate contiguous BSHD activations with a shared S×(D/2) complex cache."""
assert can_use_fused_complex_rope(x, rope)
if not (
x.is_cuda
and torch.version.hip is None
and x.dtype in (torch.float16, torch.bfloat16, torch.float32)
and x.ndim == 4
and x.numel() > 0
and x.shape[-1] % 2 == 0
and x.is_contiguous()
and rope.dtype == torch.complex64
and rope.device == x.device
and rope.shape == (x.shape[1], x.shape[-1] // 2)
and rope.is_contiguous()
):
raise RuntimeError("invalid input for fused_complex_rope")
out = torch.empty_like(x)
pairs = x.numel() // 2
with torch.cuda.device(x.device):
@@ -1,11 +1,4 @@
"""Bit-exact paired RoPE for Helios' transposed frequency layout.
The JIT kernel preserves the eager path's separate fp32 multiply and add/sub
rounding boundaries, then rounds each adjacent pair back to fp16/bf16 in
place. It is verified on head dimensions 64, 128, and 256, including Helios'
production ``[8640, 40, 128]`` Q/K shape. Unsupported layouts retain the eager
model path through :func:`can_use_helios_qk_rope`.
"""
"""Helios paired RoPE, preserving separate FP32 multiply/add rounding."""
from __future__ import annotations
@@ -22,10 +15,8 @@ if TYPE_CHECKING:
@cache_once
def _jit_helios_qk_rope_module(dtype: torch.dtype) -> Module:
if dtype not in (torch.float16, torch.bfloat16):
raise RuntimeError(
f"Unsupported Helios QK RoPE dtype {dtype}; expected float16 or bfloat16"
)
if dtype not in (torch.float16, torch.bfloat16, torch.float32):
raise RuntimeError(f"Unsupported Helios QK RoPE dtype: {dtype}")
args = make_cpp_args(dtype)
return load_jit(
"helios_qk_rope",
@@ -41,41 +32,13 @@ def fused_inplace_helios_qk_rope(
k: torch.Tensor,
freqs: torch.Tensor,
) -> None:
"""Apply Helios' transposed RoPE to contiguous normalized Q/K in place."""
"""Rotate contiguous Q/K shaped [N, H, D] or [B, S, H, D] in place.
Frequencies have shape [N, 2D] or [B, S, 2D]. The C++ launcher
validates these shapes, dtypes, devices and pair alignment.
"""
module = _jit_helios_qk_rope_module(q.dtype)
module.helios_qk_rope(q, k, freqs)
def can_use_helios_qk_rope(
q: torch.Tensor,
k: torch.Tensor,
freqs: torch.Tensor,
) -> bool:
"""Return whether tensors match the native Helios paired-RoPE contract."""
if q.dim() != 4 or freqs.dim() != 3:
return False
# Dynamo cannot trace pointer or storage-offset queries. Compiled Helios Q/K
# come directly from aligned linear outputs; eager callers retain the guard.
pair_aligned = True
if not torch.compiler.is_compiling():
pair_aligned = q.storage_offset() % 2 == 0 and k.storage_offset() % 2 == 0
return (
q.is_cuda
and k.is_cuda
and freqs.is_cuda
and q.dtype in (torch.float16, torch.bfloat16)
and k.dtype == q.dtype
and freqs.dtype is torch.float32
and q.device == k.device == freqs.device
and k.shape == q.shape
and all(size > 0 for size in q.shape)
and freqs.shape == (*q.shape[:2], 2 * q.shape[-1])
and q.shape[-1] % 2 == 0
and q.is_contiguous()
and k.is_contiguous()
and freqs.is_contiguous()
and pair_aligned
)
__all__ = ["can_use_helios_qk_rope", "fused_inplace_helios_qk_rope"]
__all__ = ["fused_inplace_helios_qk_rope"]
@@ -54,52 +54,10 @@ def fused_interleaved_rope_fp64(
q_out = torch.empty_like(q)
k_out = torch.empty_like(k)
module = _jit_interleaved_rope_fp64_module(q.dtype)
module.interleaved_rope_fp64(
q_out.view(-1),
k_out.view(-1),
q.view(-1),
k.view(-1),
cos.view(-1),
sin.view(-1),
q.shape[0],
q.shape[1],
q.shape[2],
q.shape[3],
)
module.interleaved_rope_fp64(q_out, k_out, q, k, cos, sin)
return q_out, k_out
def can_use_interleaved_rope_fp64(
q: torch.Tensor,
k: torch.Tensor,
cos: torch.Tensor,
sin: torch.Tensor,
) -> bool:
if q.dim() != 4:
return False
expected_table_shape = (1, q.shape[1], 1, q.shape[3])
return (
q.dtype is torch.bfloat16
and k.dtype is q.dtype
and q.is_cuda
and k.is_cuda
and q.device == k.device == cos.device == sin.device
and k.shape == q.shape
and q.shape[-1] % 2 == 0
and q.is_contiguous()
and k.is_contiguous()
and q.data_ptr() % 4 == 0
and k.data_ptr() % 4 == 0
and cos.dtype is torch.float64
and sin.dtype is torch.float64
and cos.shape == expected_table_shape
and sin.shape == expected_table_shape
and cos.is_contiguous()
and sin.is_contiguous()
)
__all__ = [
"can_use_interleaved_rope_fp64",
"fused_interleaved_rope_fp64",
]
@@ -67,74 +67,11 @@ def fused_ltx25_decoder_rope(
k_out = torch.empty_like(k)
module = _jit_ltx25_decoder_rope_module(q.dtype)
module.ltx25_decoder_rope(
q_out.view(-1),
k_out.view(-1),
q.view(-1),
k.view(-1),
cos_t.view(-1),
sin_t.view(-1),
cos_h.view(-1),
sin_h.view(-1),
cos_w.view(-1),
sin_w.view(-1),
q.shape[0],
q.shape[1],
q.shape[2],
q.shape[3],
q.shape[4],
q.shape[5],
dim_t,
dim_h,
q_out, k_out, q, k, cos_t, sin_t, cos_h, sin_h, cos_w, sin_w, dim_t, dim_h
)
return q_out, k_out
def can_use_ltx25_decoder_rope(
q: torch.Tensor,
k: torch.Tensor,
tables: tuple[tuple[torch.Tensor, torch.Tensor], ...],
dim_split: tuple[int, int, int],
) -> bool:
if (
q.dim() != 6
or len(tables) != 3
or any(len(pair) != 2 for pair in tables)
or len(dim_split) != 3
):
return False
dim_t, dim_h, dim_w = dim_split
expected_shapes = (
(q.shape[1], dim_t // 2),
(q.shape[2], dim_h // 2),
(q.shape[3], dim_w // 2),
)
flat_tables = tuple(table for pair in tables for table in pair)
return (
q.dtype is torch.bfloat16
and k.dtype is q.dtype
and q.is_cuda
and k.is_cuda
and q.device == k.device
and k.shape == q.shape
and all(size > 0 for size in q.shape)
and q.shape[-1] == sum(dim_split)
and all(dim > 0 and dim % 2 == 0 for dim in dim_split)
and q.is_contiguous()
and k.is_contiguous()
and q.data_ptr() % 4 == 0
and k.data_ptr() % 4 == 0
and all(table.device == q.device for table in flat_tables)
and all(table.dtype is torch.float32 for table in flat_tables)
and all(
cos.shape == sin.shape == expected_shape
and cos.is_contiguous()
and sin.is_contiguous()
for (cos, sin), expected_shape in zip(tables, expected_shapes, strict=True)
)
)
__all__ = [
"can_use_ltx25_decoder_rope",
"fused_ltx25_decoder_rope",
]
@@ -33,7 +33,6 @@ if TYPE_CHECKING:
from tvm_ffi.module import Module
_HEAD_DIM = 128
_ROW_ALIGN = 8
@cache_once
@@ -50,117 +49,19 @@ def _jit_module(fuse_real_sin: bool, copy_v: bool) -> Module:
)
def _is_rows(t: torch.Tensor, device: torch.device, tokens: int | None = None) -> bool:
"""``[B, T, H, 128]`` bf16 CUDA tensor whose heads are contiguous; batch/token strides may be padded."""
return (
isinstance(t, torch.Tensor)
and t.dtype is torch.bfloat16
and t.is_cuda
and t.device == device
and t.dim() == 4
and t.numel() > 0
and t.shape[-1] == _HEAD_DIM
and (tokens is None or t.shape[1] == tokens)
and t.stride(-1) == 1
and t.stride(-2) == _HEAD_DIM
and t.stride(1) % (_ROW_ALIGN // 2) == 0
and t.stride(0) % (_ROW_ALIGN // 2) == 0
and t.data_ptr() % _ROW_ALIGN == 0
)
def _is_weight(w: torch.Tensor, device: torch.device) -> bool:
return (
isinstance(w, torch.Tensor)
and w.dtype is torch.bfloat16
and w.is_cuda
and w.device == device
and w.shape == (_HEAD_DIM,)
and w.is_contiguous()
)
def _is_rope(rope: torch.Tensor, seq: int, device: torch.device) -> bool:
return (
isinstance(rope, torch.Tensor)
and rope.dtype is torch.complex64
and rope.device == device
and rope.shape == (seq, _HEAD_DIM // 2)
and rope.is_contiguous()
)
def _cuda_platform(t: torch.Tensor) -> bool:
return t.is_cuda and torch.version.hip is None and not torch.compiler.is_compiling()
@torch.compiler.assume_constant_result
@cache_once
def can_use_qknorm_complex_rope_cuda(dtype: torch.dtype, head_dim: int) -> bool:
"""The CUDA reduction is specialized to BF16, 128-wide heads."""
return dtype is torch.bfloat16 and head_dim == _HEAD_DIM
def _rope_real(rope: torch.Tensor) -> torch.Tensor:
return torch.view_as_real(rope).view(rope.shape[0], _HEAD_DIM)
def can_use_qknorm_complex_rope_cuda(
x: torch.Tensor, weight: torch.Tensor, rope: torch.Tensor
) -> bool:
"""Contiguous ``[B, S, H, 128]`` bf16 Q or K with its ``(S, 64)`` complex64 table."""
return (
_cuda_platform(x)
and _is_rows(x, x.device)
and x.is_contiguous()
and _is_weight(weight, x.device)
and _is_rope(rope, x.shape[1], x.device)
)
def can_use_qknorm_complex_rope_pack(
q: torch.Tensor,
k_out: torch.Tensor,
v_out: torch.Tensor,
q_weight: torch.Tensor,
k_weight: torch.Tensor,
rope: torch.Tensor,
k_prefix: torch.Tensor,
v_prefix: torch.Tensor,
k_src: Optional[torch.Tensor],
v_src: Optional[torch.Tensor],
) -> bool:
"""Shapes for :func:`qknorm_complex_rope_pack_`; ``None`` sources mean "already in the buffer"."""
if not (_cuda_platform(q) and _is_rows(q, q.device)):
return False
device = q.device
batch, seq, heads, _ = q.shape
if not (
k_prefix.dim() == 4
and k_prefix.shape[0] == batch
and k_prefix.shape[1] > 0
and k_prefix.shape[2] == heads
and k_prefix.shape[3] == _HEAD_DIM
and k_prefix.is_contiguous()
and v_prefix.shape == k_prefix.shape
and v_prefix.is_contiguous()
and all(
t.dtype is torch.bfloat16 and t.device == device
for t in (k_prefix, v_prefix)
)
):
return False
prefix = k_prefix.shape[1]
return (
_is_rows(k_out, device, prefix + seq)
and _is_rows(v_out, device, prefix + seq)
and k_out.shape[0] == batch
and v_out.shape[0] == batch
and k_out.shape[2] == heads
and v_out.shape[2] == heads
and (k_src is None or (_is_rows(k_src, device, seq) and k_src.shape == q.shape))
and (v_src is None or (_is_rows(v_src, device, seq) and v_src.shape == q.shape))
and _is_weight(q_weight, device)
and _is_weight(k_weight, device)
and _is_rope(rope, seq, device)
)
return torch.view_as_real(rope)
def _fake_norm_rope(x, weight, rope, eps):
return torch.empty_like(x)
return torch.empty(x.shape, dtype=x.dtype, device=x.device)
@register_custom_op(
@@ -171,8 +72,8 @@ def _fake_norm_rope(x, weight, rope, eps):
def qknorm_complex_rope_cuda(
x: torch.Tensor, weight: torch.Tensor, rope: torch.Tensor, eps: float
) -> torch.Tensor:
"""``rope(rmsnorm(x))`` for one contiguous ``[B, S, H, 128]`` tensor, out of place."""
out = torch.empty_like(x)
"""``rope(rmsnorm(x))`` for one ``[B, S, H, 128]`` tensor with packed head rows, out of place."""
out = torch.empty(x.shape, dtype=x.dtype, device=x.device)
module = _jit_module(_fuse_real_sin(x.device), False)
with torch.cuda.device(x.device):
module.norm_rope(x, out, weight, _rope_real(rope), float(eps))
@@ -224,7 +125,6 @@ def qknorm_complex_rope_pack_(
__all__ = [
"can_use_qknorm_complex_rope_cuda",
"can_use_qknorm_complex_rope_pack",
"qknorm_complex_rope_cuda",
"qknorm_complex_rope_pack_",
]
@@ -8,7 +8,7 @@ import triton.language as tl
from sglang.kernels.ops.diffusion.rope.complex_rope_triton import _fuse_real_sin
from sglang.kernels.ops.diffusion.rope.qknorm_complex_rope_triton import (
_qknorm_complex_rope_rows,
can_use_qknorm_complex_rope,
_validate_qknorm_complex_rope,
)
from sglang.srt.utils.custom_op import register_custom_op
@@ -54,22 +54,6 @@ def _qknorm_complex_rope_kv_kernel(
tl.store(vout_ptr + prefix_index, prefix_value, prefix_mask)
def can_use_qknorm_complex_rope_kv(k, weight, rope, v, k_prefix, v_prefix):
return (
can_use_qknorm_complex_rope(k, weight, rope)
and v.shape == k.shape
and k_prefix.ndim == 4
and k_prefix.shape[0] == k.shape[0]
and k_prefix.shape[1] > 0
and k_prefix.shape[2:] == k.shape[2:]
and v_prefix.shape == k_prefix.shape
and all(
x.device == k.device and x.dtype == k.dtype and x.is_contiguous()
for x in (v, k_prefix, v_prefix)
)
)
def _fake_qknorm_complex_rope_kv(k, weight, rope, v, k_prefix, v_prefix, eps):
shape = (k.shape[0], k_prefix.shape[1] + k.shape[1], *k.shape[2:])
return k.new_empty(shape), v.new_empty(shape)
@@ -89,7 +73,21 @@ def qknorm_complex_rope_kv(
v_prefix: torch.Tensor,
eps: float,
) -> tuple[torch.Tensor, torch.Tensor]:
assert can_use_qknorm_complex_rope_kv(k, weight, rope, v, k_prefix, v_prefix)
k, v = k.contiguous(), v.contiguous()
_validate_qknorm_complex_rope(k, weight, rope)
if not (
v.shape == k.shape
and k_prefix.ndim == 4
and k_prefix.shape[0] == k.shape[0]
and k_prefix.shape[1] > 0
and k_prefix.shape[2:] == k.shape[2:]
and v_prefix.shape == k_prefix.shape
and all(
x.device == k.device and x.dtype == k.dtype and x.is_contiguous()
for x in (v, k_prefix, v_prefix)
)
):
raise RuntimeError("invalid input for qknorm_complex_rope_kv")
kout, vout = _fake_qknorm_complex_rope_kv(
k, weight, rope, v, k_prefix, v_prefix, eps
)
@@ -5,12 +5,8 @@ import torch
import triton
import triton.language as tl
from sglang.kernels.ops.diffusion.norm.rmsnorm_preserve_reduction import (
can_use_rmsnorm_preserve_reduction,
)
from sglang.kernels.ops.diffusion.rope.complex_rope_triton import (
_fuse_real_sin,
can_use_fused_complex_rope,
)
from sglang.srt.utils.custom_op import register_custom_op
@@ -80,16 +76,29 @@ def _qknorm_complex_rope_onepass_kernel(
)
def can_use_qknorm_complex_rope(x, weight, rope):
return (
can_use_rmsnorm_preserve_reduction(x, weight)
and can_use_fused_complex_rope(x, rope)
def _validate_qknorm_complex_rope(x, weight, rope):
if not (
x.is_cuda
and torch.version.hip is None
and x.dtype in (torch.float16, torch.bfloat16)
and x.ndim == 4
and x.shape[-1] == 128
)
and x.numel() > 0
and x.is_contiguous()
and weight.device == x.device
and weight.dtype == x.dtype
and weight.shape == (128,)
and weight.is_contiguous()
and rope.dtype == torch.complex64
and rope.device == x.device
and rope.shape == (x.shape[1], 64)
and rope.is_contiguous()
):
raise RuntimeError("invalid input for qknorm_complex_rope")
def _fake_qknorm_complex_rope(x, weight, rope, eps):
return torch.empty_like(x)
return torch.empty(x.shape, dtype=x.dtype, device=x.device)
@register_custom_op(
@@ -103,7 +112,8 @@ def qknorm_complex_rope(
rope: torch.Tensor,
eps: float,
) -> torch.Tensor:
assert can_use_qknorm_complex_rope(x, weight, rope)
x = x.contiguous()
_validate_qknorm_complex_rope(x, weight, rope)
out = torch.empty_like(x)
with torch.cuda.device(x.device):
_qknorm_complex_rope_onepass_kernel[(triton.cdiv(x.numel() // 128, 4),)](
@@ -83,32 +83,6 @@ def _rope_rotate_half_kernel(
tl.store(out_ptr + toff, tail, mask=tmask)
def can_use_fused_rope_rotate_half(
x: torch.Tensor,
cos: torch.Tensor,
sin: torch.Tensor,
) -> bool:
if x.dtype is not torch.bfloat16 or not x.is_cuda:
return False
if x.dim() != 4 or not x.is_contiguous():
return False
rows = x.shape[0] * x.shape[1]
rot = cos.shape[-1]
return (
cos.dtype is torch.bfloat16
and sin.dtype is torch.bfloat16
and cos.is_cuda
and cos.device == x.device
and sin.device == x.device
and cos.shape == (rows, rot)
and sin.shape == (rows, rot)
and cos.is_contiguous()
and sin.is_contiguous()
and rot % 2 == 0
and 0 < rot <= x.shape[-1]
)
def _fake_rope_rotate_half(
x: torch.Tensor,
cos: torch.Tensor,
@@ -129,15 +103,36 @@ def fused_rope_rotate_half_bitexact(
) -> torch.Tensor:
"""Rotate-half RoPE over the leading ``cos.shape[-1]`` columns of ``x``.
``x`` is ``(B, S, H, D)``; ``cos``/``sin`` are ``(B * S, rot_dim)`` rows.
``x`` is ``(B, S, H, D)``; ``cos``/``sin`` are contiguous
``(B * S, rot_dim)`` rows or ``(B, S, 1, rot_dim)`` broadcast tables.
Bit-exact vs the eager chunk/neg/cat/mul/add chain.
"""
if not (
x.is_cuda and x.dtype is torch.bfloat16 and x.ndim == 4 and x.is_contiguous()
):
raise RuntimeError("rotate-half RoPE expects contiguous BF16 CUDA [B, S, H, D]")
batch, seq_len, heads, head_dim = x.shape
if cos.ndim not in (2, 4):
raise RuntimeError("cos must have shape [B * S, rot_dim] or [B, S, 1, rot_dim]")
rot = cos.shape[-1]
if not (0 < rot <= head_dim and rot % 2 == 0):
raise RuntimeError("rot_dim must be positive, even and no larger than head_dim")
table_shape = (batch * seq_len, rot) if cos.ndim == 2 else (batch, seq_len, 1, rot)
device = x.device
for name, tensor in (("cos", cos), ("sin", sin)):
if not (
tensor.dtype is torch.bfloat16
and tensor.device == device
and tensor.shape == table_shape
and tensor.is_contiguous()
):
raise RuntimeError(
f"{name} must be contiguous {table_shape} with x's dtype/device"
)
half = rot // 2
out = torch.empty_like(x)
tail = head_dim - rot
with torch.cuda.device(x.device):
with torch.cuda.device(device):
_rope_rotate_half_kernel[(batch * seq_len,)](
out,
x,
@@ -22,7 +22,6 @@ import torch
from torch import nn
from sglang.kernels.ops.diffusion.norm.rmsnorm_scale_shift_bitexact import (
can_use_fused_rmsnorm_scale_shift,
fused_rmsnorm_scale_shift_bitexact,
)
from sglang.kernels.ops.diffusion.sites.quality_gate import QualityGatedFusion
@@ -65,14 +64,6 @@ def _ones_weight(x: torch.Tensor) -> torch.Tensor:
return w
def can_use_ltx2_rms_norm_modulate(
x: torch.Tensor, scale: torch.Tensor, shift: torch.Tensor
) -> bool:
if x.dtype is not torch.bfloat16 or not x.is_cuda:
return False
return can_use_fused_rmsnorm_scale_shift(x, _ones_weight(x), scale, shift)
def fused_ltx2_rms_norm_modulate(
x: torch.Tensor, scale: torch.Tensor, shift: torch.Tensor, eps: float
) -> torch.Tensor:
@@ -19,7 +19,6 @@ from torch import nn
from sglang.kernels.ops.diffusion import (
BitExactFusionGate,
can_use_ltx25_decoder_rope,
fused_ltx25_decoder_rope,
tensors_equal,
)
@@ -32,8 +31,11 @@ from sglang.multimodal_gen.runtime.layers.visual_embedding import (
from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import (
LayerwiseOffloadableModuleMixin,
)
from sglang.multimodal_gen.runtime.platforms import current_platform
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
_is_cuda = current_platform.is_cuda()
logger = init_logger(__name__)
# Bounds the three hidden-width temporaries the SwiGLU holds live. Exact: the
@@ -326,7 +328,9 @@ class LTX2VideoVaeRotaryPosEmbed3D(nn.Module):
verified = _LTX25_DECODER_ROPE.verified
if (
not _LTX25_DECODER_ROPE.disabled
and can_use_ltx25_decoder_rope(query, key, tables, self.rope_dim_split)
and _is_cuda
and query.is_cuda
and query.dtype is torch.bfloat16
and (verified or _LTX25_DECODER_ROPE.can_attempt_once())
):
dim_t, dim_h, _ = self.rope_dim_split
@@ -25,8 +25,6 @@ from sglang.kernels.ops.activation.activation import (
from sglang.kernels.ops.diffusion import (
BitExactFusionGate,
can_use_fused_rmsnorm_scale_shift,
can_use_fused_rope_rotate_half,
can_use_fused_scale_residual_rmsnorm_scale_shift,
flashinfer_rmsnorm_diagnostic_hint,
fused_rmsnorm_scale_shift_bitexact,
fused_rope_rotate_half_bitexact,
@@ -92,8 +90,9 @@ def _ernie_norm_scale_shift(
verified = _ERNIE_NORM.verified
if (
not _ERNIE_NORM.disabled
and x.is_cuda
and norm.variance_size_override is None
and can_use_fused_rmsnorm_scale_shift(x, norm.weight, scale, shift)
and can_use_fused_rmsnorm_scale_shift(x.dtype, x.shape[-1])
and (verified or _ERNIE_NORM.can_attempt_once())
):
try:
@@ -136,10 +135,9 @@ def _ernie_gated_norm_scale_shift(
verified = _ERNIE_GATED_NORM.verified
if (
not _ERNIE_GATED_NORM.disabled
and residual.is_cuda
and norm.variance_size_override is None
and can_use_fused_scale_residual_rmsnorm_scale_shift(
residual, update, gate, norm.weight, scale, shift
)
and can_use_fused_rmsnorm_scale_shift(residual.dtype, residual.shape[-1])
and (verified or _ERNIE_GATED_NORM.can_attempt_once())
):
try:
@@ -461,7 +459,8 @@ def _ernie_rope(
verified = _ERNIE_ROPE.verified
if (
not _ERNIE_ROPE.disabled
and can_use_fused_rope_rotate_half(x, cos_, sin_)
and x.is_cuda
and x.dtype is torch.bfloat16
and (verified or _ERNIE_ROPE.can_attempt_once())
):
try:
@@ -122,8 +122,9 @@ def _flux_fused_ln_modulate(
"""
if (
_FLUX_LN_MOD.disabled
or not x.is_cuda
or not is_plain_layer_norm(norm, x.shape[-1])
or not can_use_fused_layernorm_modulate(x, scale, shift)
or not can_use_fused_layernorm_modulate(x.dtype, x.shape[-1])
):
return None
sig = (
@@ -153,7 +153,7 @@ def _norm_modulate(
_LN_MODULATE.disabled
or not _eager_fast_path_allowed(x)
or not is_plain_layer_norm(norm, x.shape[-1])
or not can_use_fused_layernorm_modulate(x, scale_row, shift_row)
or not can_use_fused_layernorm_modulate(x.dtype, x.shape[-1])
):
return (1 + scale) * norm(x) + shift
sig = (x.device, x.shape[0], x.shape[-1], norm.eps)
@@ -208,9 +208,10 @@ def _try_flux2_norm_modulate_fp8(
shift_row = shift.squeeze(1) if shift.dim() == 3 and shift.shape[1] == 1 else shift
if (
_FLUX2_LN_FP8.disabled
or not x.is_cuda
or x.shape[-1] != 6144
or not is_plain_layer_norm(norm, x.shape[-1])
or not can_use_fused_layernorm_modulate(x, scale_row, shift_row)
or not can_use_fused_layernorm_modulate(x.dtype, x.shape[-1])
):
return None
@@ -373,8 +374,9 @@ def _flux2_norm_modulate(
shift_row = shift.squeeze(1) if shift.dim() == 3 and shift.shape[1] == 1 else shift
if (
_FLUX2_LN_MOD.disabled
or not x.is_cuda
or not is_plain_layer_norm(norm, x.shape[-1])
or not can_use_fused_layernorm_modulate(x, scale_row, shift_row)
or not can_use_fused_layernorm_modulate(x.dtype, x.shape[-1])
):
return norm(x) * (1 + scale) + shift
@@ -103,9 +103,10 @@ def _glm_ln_modulate(
if (
not _GLM_LN_MOD.disabled
and _is_cuda
and x.is_cuda
and dtype is x.dtype
and is_plain_layer_norm(norm, x.shape[-1])
and can_use_fused_layernorm_modulate(x, scale, shift)
and can_use_fused_layernorm_modulate(x.dtype, x.shape[-1])
and (verified or _GLM_LN_MOD.can_attempt_once())
):
try:
@@ -144,12 +145,13 @@ def _glm_qk_layernorm(
if (
not _GLM_QK_LN.disabled
and _is_cuda
and query.is_cuda
and dtype is query.dtype
and dtype is key.dtype
and is_plain_layer_norm(norm_q, query.shape[-1])
and is_plain_layer_norm(norm_k, key.shape[-1])
and norm_q.eps == norm_k.eps
and can_use_fused_qk_head_layernorm(query, key)
and can_use_fused_qk_head_layernorm(query.dtype, query.shape[-1])
and (verified or _GLM_QK_LN.can_attempt_once())
):
try:
@@ -17,7 +17,6 @@ import torch.nn as nn
import torch.nn.functional as F
from sglang.kernels.ops.diffusion import (
can_use_helios_qk_rope,
fused_inplace_helios_qk_rope,
mark_helios_gated_residual_site,
try_helios_gated_residual,
@@ -58,8 +57,11 @@ from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload im
LayerwiseOffloadableModuleMixin,
)
from sglang.multimodal_gen.runtime.models.dits.base import CachableDiT
from sglang.multimodal_gen.runtime.platforms import current_platform
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
_is_cuda = current_platform.is_cuda()
logger = init_logger(__name__)
@@ -297,12 +299,8 @@ class HeliosSelfAttention(nn.Module):
k: torch.Tensor,
rotary_emb: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
if not self.tp_rmsnorm and can_use_helios_qk_rope(q, k, rotary_emb):
fused_inplace_helios_qk_rope(
q.view(-1, q.shape[-2], q.shape[-1]),
k.view(-1, k.shape[-2], k.shape[-1]),
rotary_emb.view(-1, rotary_emb.shape[-1]),
)
if not self.tp_rmsnorm and _is_cuda and q.is_cuda:
fused_inplace_helios_qk_rope(q, k, rotary_emb)
return q, k
return (
apply_rotary_emb_transposed(q, rotary_emb),
@@ -9,7 +9,6 @@ import torch.nn.functional as F
from sglang.kernels.ops.diffusion import (
BitExactFusionGate,
can_use_fused_silu_mul,
fused_gate_rmsnorm_active,
fused_rmsnorm_scale,
fused_rmsnorm_tanh_residual,
@@ -54,9 +53,14 @@ from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload im
LayerwiseOffloadableModuleMixin,
)
from sglang.multimodal_gen.runtime.models.dits.base import BaseDiT
from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum
from sglang.multimodal_gen.runtime.platforms import (
AttentionBackendEnum,
current_platform,
)
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
_is_cuda = current_platform.is_cuda()
logger = init_logger(__name__)
OUTPUT_IMAGE_INDICATOR = 2
@@ -67,35 +71,6 @@ _IDEOGRAM_SWIGLU = BitExactFusionGate("Ideogram fused SiLU-mul")
_IDEOGRAM_ZERO_SHIFTS: dict[tuple[torch.device, torch.dtype, int], torch.Tensor] = {}
def _can_use_fused_rope(
q: torch.Tensor,
k: torch.Tensor,
cos: torch.Tensor,
sin: torch.Tensor,
) -> bool:
# cos/sin are full-span (B, S, 1, D) rows broadcast over heads.
expected = (q.shape[0], q.shape[1], 1, q.shape[-1])
return (
q.dtype is torch.bfloat16
and k.dtype is torch.bfloat16
and q.is_cuda
and q.dim() == 4
and q.is_contiguous()
and k.shape == q.shape
and k.is_contiguous()
and k.device == q.device
and cos.dtype is torch.bfloat16
and sin.dtype is torch.bfloat16
and cos.device == q.device
and sin.device == q.device
and cos.shape == expected
and sin.shape == expected
and cos.is_contiguous()
and sin.is_contiguous()
and q.shape[-1] % 2 == 0
)
def _ideogram_rope(
q: torch.Tensor,
k: torch.Tensor,
@@ -113,14 +88,15 @@ def _ideogram_rope(
verified = _IDEOGRAM_ROPE.verified
if (
not _IDEOGRAM_ROPE.disabled
and _can_use_fused_rope(q, k, cos, sin)
and q.is_cuda
and q.dtype is torch.bfloat16
and q.is_contiguous()
and k.is_contiguous()
and (verified or _IDEOGRAM_ROPE.can_attempt_once())
):
try:
cos_rows = cos.reshape(-1, cos.shape[-1])
sin_rows = sin.reshape(-1, sin.shape[-1])
q_fused = fused_rope_rotate_half_bitexact(q, cos_rows, sin_rows)
k_fused = fused_rope_rotate_half_bitexact(k, cos_rows, sin_rows)
q_fused = fused_rope_rotate_half_bitexact(q, cos, sin)
k_fused = fused_rope_rotate_half_bitexact(k, cos, sin)
except Exception as exc:
_IDEOGRAM_ROPE.on_exception(exc, logger=logger)
else:
@@ -144,7 +120,9 @@ def _ideogram_swiglu(a: torch.Tensor, b: torch.Tensor) -> torch.Tensor:
verified = _IDEOGRAM_SWIGLU.verified
if (
not _IDEOGRAM_SWIGLU.disabled
and can_use_fused_silu_mul(a, b)
and _is_cuda
and a.is_cuda
and a.dtype is torch.bfloat16
and (verified or _IDEOGRAM_SWIGLU.can_attempt_once())
):
try:
@@ -177,7 +177,9 @@ def _joy_joint_qkv(*inputs: torch.Tensor) -> tuple[torch.Tensor, ...]:
or not inputs[0].is_cuda
or torch.version.hip
or torch.compiler.is_compiling()
or not diffusion_kernels.can_use_joint_qkv_cat(*inputs)
or inputs[0].dtype not in (torch.float16, torch.bfloat16)
or inputs[0].shape[-2] * inputs[0].shape[-1] > 8192
or any(x.requires_grad for x in inputs)
):
return reference()
sig = (inputs[0].device, inputs[0].dtype) + tuple(
@@ -58,8 +58,11 @@ from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload im
LayerwiseOffloadableModuleMixin,
)
from sglang.multimodal_gen.runtime.models.dits.base import BaseDiT
from sglang.multimodal_gen.runtime.platforms import current_platform
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
_is_cuda = current_platform.is_cuda()
logger = init_logger(__name__)
_LONGCAT_QKNORM_ROPE = BitExactFusionGate("LongCat fused QKNorm+RoPE")
@@ -74,7 +77,11 @@ def _longcat_gelu_cat(
type(activation) is nn.GELU
and activation.approximate == "tanh"
and _LONGCAT_GELU_CAT.can_attempt_once()
and diffusion_ops.can_use_fused_gelu_tanh_cat(attn, mlp)
and _is_cuda
and attn.is_cuda
and attn.dtype is torch.bfloat16
and not torch.is_grad_enabled()
and not torch.compiler.is_compiling()
):
try:
output = diffusion_ops.fused_gelu_tanh_cat(attn, mlp)
@@ -104,7 +111,7 @@ def _longcat_norm_modulate(
not _LONGCAT_LN_MOD.disabled
and x.is_cuda
and diffusion_ops.is_plain_layer_norm(norm, x.shape[-1])
and diffusion_ops.can_use_fused_layernorm_modulate(x, scale, shift)
and diffusion_ops.can_use_fused_layernorm_modulate(x.dtype, x.shape[-1])
):
sig = (
x.shape,
@@ -12,9 +12,9 @@ import torch.nn.functional as F
from sglang.kernels.ops.diffusion import (
BitExactFusionGate,
can_use_fused_rmsnorm_scale_shift,
can_use_linear_gelu,
can_use_ltx2_qknorm_split_rope_cuda,
can_use_ltx2_rms_norm_modulate,
can_use_modulate_scale_shift_cuda,
fused_gelu_active,
fused_linear_gelu_tanh,
@@ -208,8 +208,10 @@ def _ltx2_rms_norm_modulate(
default). The fused kernel is not bit-exact (<=1 bf16 ULP) so it is gated
on the request-scoped mount rather than a runtime self-check.
"""
if ltx2_rms_norm_modulate_active(block) and can_use_ltx2_rms_norm_modulate(
x, scale, shift
if (
ltx2_rms_norm_modulate_active(block)
and x.is_cuda
and can_use_fused_rmsnorm_scale_shift(x.dtype, x.shape[-1])
):
return fused_ltx2_rms_norm_modulate(x, scale, shift, eps)
normed = rms_norm(x, eps)
@@ -520,7 +520,7 @@ def delta_factor_apply(
B32.contiguous(),
alpha.float().contiguous(),
)
if can_use_vdn_delta_factors(A32, B32, alpha32):
if can_use_vdn_delta_factors(A32.device, A32.dtype, A32.shape[-1]):
return vdn_delta_factors(A32, B32, alpha32)
chol = torch.linalg.cholesky(A32 + eye)
# (I+A)^-1 = L^-T L^-1: a batched trsm at 128x128 is far slower than the GEMM
@@ -133,8 +133,11 @@ def _qwen_norm_out(
scale, shift = torch.chunk(emb, 2, dim=1)
if (
_QWEN_NORM_OUT.disabled
or not hidden_states.is_cuda
or not is_plain_layer_norm(norm_out.norm, hidden_states.shape[-1])
or not can_use_fused_layernorm_modulate(hidden_states, scale, shift)
or not can_use_fused_layernorm_modulate(
hidden_states.dtype, hidden_states.shape[-1]
)
):
return (
norm_out.norm(hidden_states) * (1 + scale)[:, None, :] + shift[:, None, :]
@@ -8,12 +8,8 @@ from torch import nn
from sglang.kernels.ops.diffusion import (
BitExactFusionGate,
can_use_fused_complex_rope,
can_use_fused_layernorm_modulate,
can_use_fused_silu_mul,
can_use_qknorm_complex_rope_cuda,
can_use_qknorm_complex_rope_pack,
can_use_rmsnorm_preserve_reduction,
fused_complex_rope,
fused_layernorm_modulate,
fused_silu_mul_bitexact,
@@ -24,11 +20,9 @@ from sglang.kernels.ops.diffusion import (
tensors_equal,
)
from sglang.kernels.ops.diffusion.rope.qknorm_complex_rope_kv_triton import (
can_use_qknorm_complex_rope_kv,
qknorm_complex_rope_kv,
)
from sglang.kernels.ops.diffusion.rope.qknorm_complex_rope_triton import (
can_use_qknorm_complex_rope,
qknorm_complex_rope,
)
from sglang.multimodal_gen.runtime.distributed import (
@@ -51,10 +45,15 @@ from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload im
LayerwiseOffloadableModuleMixin,
)
from sglang.multimodal_gen.runtime.models.dits.base import CachableDiT
from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum
from sglang.multimodal_gen.runtime.platforms import (
AttentionBackendEnum,
current_platform,
)
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
from sglang.srt.layers.layernorm import RMSNorm
_is_cuda = current_platform.is_cuda()
logger = init_logger(__name__)
_ROPE_FUSION = BitExactFusionGate("Qwen-Image 2.1 complex RoPE")
_SILU_MUL_FUSION = BitExactFusionGate("Qwen-Image 2.1 SiLU-mul")
@@ -130,7 +129,7 @@ def build_layout(image_slots, image_shapes, axes_dims, device):
def apply_rope(x, rope):
fused = None
if can_use_fused_complex_rope(x, rope) and _ROPE_FUSION.can_attempt_once():
if _is_cuda and x.is_cuda and _ROPE_FUSION.can_attempt_once():
fused = fused_complex_rope(x, rope)
if _ROPE_FUSION.verified:
return fused
@@ -144,7 +143,9 @@ def apply_rope(x, rope):
def apply_qk_norm(x, norm):
fused = None
if (
can_use_rmsnorm_preserve_reduction(x, norm.weight)
_is_cuda
and x.is_cuda
and x.dtype in (torch.float16, torch.bfloat16)
and _QK_NORM_FUSION.can_attempt_once()
):
fused = rmsnorm_preserve_reduction(x, norm.weight, norm.variance_epsilon)
@@ -159,22 +160,28 @@ def apply_qk_norm(x, norm):
def apply_qk_norm_rope(x, norm, rope):
fused = None
if (
can_use_qknorm_complex_rope_cuda(x, norm.weight, rope)
_is_cuda
and x.is_cuda
and not torch.compiler.is_compiling()
and can_use_qknorm_complex_rope_cuda(x.dtype, x.shape[-1])
and _QK_ROPE_CUDA_FUSION.can_attempt_once()
):
fused = qknorm_complex_rope_cuda(x, norm.weight, rope, norm.variance_epsilon)
if _QK_ROPE_CUDA_FUSION.verified:
return fused
out = apply_rope(apply_qk_norm(x, norm), rope)
out = apply_rope(norm(x), rope)
return _QK_ROPE_CUDA_FUSION.accept_or_fallback(fused, out, logger=logger)
if (
can_use_qknorm_complex_rope(x, norm.weight, rope)
_is_cuda
and x.is_cuda
and x.dtype in (torch.float16, torch.bfloat16)
and x.shape[-1] == 128
and _QK_ROPE_FUSION.can_attempt_once()
):
fused = qknorm_complex_rope(x, norm.weight, rope, norm.variance_epsilon)
if _QK_ROPE_FUSION.verified:
return fused
out = apply_rope(apply_qk_norm(x, norm), rope)
out = apply_rope(norm(x), rope)
if fused is not None:
return _QK_ROPE_FUSION.accept_or_fallback(fused, out, logger=logger)
return out
@@ -218,7 +225,8 @@ def cat_outputs(outputs, dim=0):
def apply_modulation(x, norm, scale):
fused = None
if (
can_use_fused_layernorm_modulate(x, scale.squeeze(1), None)
x.is_cuda
and can_use_fused_layernorm_modulate(x.dtype, x.shape[-1])
and _MODULATION_FUSION.can_attempt_once()
):
fused = fused_layernorm_modulate(x, scale.squeeze(1), None, norm.eps)
@@ -305,7 +313,11 @@ class QwenImage21FeedForward(nn.Module):
def forward(self, x):
gate, value = self.gate_layer(x)[0], self.proj(x)[0]
fused = None
if can_use_fused_silu_mul(gate, value) and _SILU_MUL_FUSION.can_attempt_once():
if (
_is_cuda
and gate.dtype is torch.bfloat16
and _SILU_MUL_FUSION.can_attempt_once()
):
fused = fused_silu_mul_bitexact(gate, value)
if _SILU_MUL_FUSION.verified:
return self.out(fused)[0]
@@ -443,9 +455,16 @@ class QwenImage21Attention(nn.Module):
With ``k_out``/``v_out`` the projection already wrote the raw K/V into
rows ``[P:]`` (``k``/``v`` are those views), so K is normalized in place
and V is not copied. Returns ``(q, k_out, v_out)`` or ``None`` when the
layout is unsupported. The first call also runs the eager chain and
dtype/head dimension is unsupported. The first call also runs the eager chain and
keeps its result on a mismatch.
"""
if not (
_is_cuda
and q.is_cuda
and not torch.compiler.is_compiling()
and can_use_qknorm_complex_rope_cuda(q.dtype, self.head_dim)
):
return None
eps = self.norm_k.variance_epsilon
if self.norm_q.variance_epsilon != eps:
return None
@@ -456,24 +475,11 @@ class QwenImage21Attention(nn.Module):
k_src, v_src = k, v
else:
k_src, v_src = None, None
if not can_use_qknorm_complex_rope_pack(
q,
k_out,
v_out,
self.norm_q.weight,
self.norm_k.weight,
rope,
kp,
vp,
k_src,
v_src,
):
return None
reference = None
if not _KV_PACK_CUDA_FUSION.verified:
reference = (
apply_rope(apply_qk_norm(q, self.norm_q), rope),
torch.cat([kp, apply_rope(apply_qk_norm(k, self.norm_k), rope)], 1),
apply_rope(self.norm_q(q), rope),
torch.cat([kp, apply_rope(self.norm_k(k), rope)], 1),
torch.cat([vp, v], 1),
)
qknorm_complex_rope_pack_(
@@ -612,7 +618,10 @@ class QwenImage21Attention(nn.Module):
packed = None
if (
get_sp_world_size() == 1
and can_use_qknorm_complex_rope_kv(k, self.norm_k.weight, rope, v, kp, vp)
and _is_cuda
and k.is_cuda
and k.dtype in (torch.float16, torch.bfloat16)
and self.head_dim == 128
and _KV_ROPE_FUSION.can_attempt_once()
):
packed = qknorm_complex_rope_kv(
@@ -620,7 +629,7 @@ class QwenImage21Attention(nn.Module):
)
if not _KV_ROPE_FUSION.verified:
reference = (
torch.cat([kp, apply_rope(apply_qk_norm(k, self.norm_k), rope)], 1),
torch.cat([kp, apply_rope(self.norm_k(k), rope)], 1),
torch.cat([vp, v], 1),
)
packed = _KV_ROPE_FUSION.accept_or_fallback(
@@ -7,8 +7,6 @@ from diffusers.models.embeddings import PixArtAlphaTextProjection, TimestepEmbed
from sglang.kernels.ops.diffusion import (
BitExactFusionGate,
can_use_fused_bias_glu,
can_use_fused_bias_silu,
can_use_fused_layernorm_modulate,
fused_bias_glu,
fused_bias_silu,
@@ -106,7 +104,7 @@ def sana_ln_modulate(
and is_plain_layer_norm(norm, x.shape[-1])
):
x_c = x.contiguous()
if not can_use_fused_layernorm_modulate(x_c, scale[:, 0], shift[:, 0]):
if not can_use_fused_layernorm_modulate(x_c.dtype, x_c.shape[-1]):
return _eager_ln_modulate(norm, x, scale, shift)
try:
out = fused_layernorm_modulate_raw(x_c, scale[:, 0], shift[:, 0], norm.eps)
@@ -185,7 +183,7 @@ def sana_conv_bias_silu(
return F.silu(_mps_safe_conv2d(conv, x))
raw = _conv2d_without_bias(conv, x)
if not can_use_fused_bias_silu(raw, conv.bias):
if raw.dtype is not torch.bfloat16:
return F.silu(raw + conv.bias[None, :, None, None])
verified = _SANA_CONV_SILU.verified
if not verified and not _SANA_CONV_SILU.can_attempt_once():
@@ -217,10 +215,8 @@ def sana_conv_bias_glu(
return hidden_states * F.silu(gate)
raw = _conv2d_without_bias(conv, x)
if not can_use_fused_bias_glu(raw, conv.bias):
hidden_states, gate = torch.chunk(
raw + conv.bias[None, :, None, None], 2, dim=1
)
if raw.dtype is not torch.bfloat16:
hidden_states, gate = (raw + conv.bias[None, :, None, None]).chunk(2, dim=1)
return hidden_states * F.silu(gate)
verified = _SANA_CONV_GLU.verified
if not verified and not _SANA_CONV_GLU.can_attempt_once():
@@ -11,7 +11,6 @@ from diffusers.models.embeddings import PixArtAlphaTextProjection
from sglang.kernels.ops.diffusion import (
BitExactFusionGate,
can_use_interleaved_rope_fp64,
fused_interleaved_rope_fp64,
mark_sana_video_linear_attention_site,
tensors_equal,
@@ -34,8 +33,11 @@ from sglang.multimodal_gen.runtime.models.dits.sana import (
sana_ln_modulate,
sana_residual_gate_add,
)
from sglang.multimodal_gen.runtime.platforms import current_platform
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
_is_cuda = current_platform.is_cuda()
logger = init_logger(__name__)
_SANA_VIDEO_ROPE = BitExactFusionGate("SANA-Video fused RoPE")
@@ -66,7 +68,9 @@ def apply_interleaved_rotary_emb_pair(
verified = _SANA_VIDEO_ROPE.verified
if (
not _SANA_VIDEO_ROPE.disabled
and can_use_interleaved_rope_fp64(query, key, freqs_cos, freqs_sin)
and _is_cuda
and query.is_cuda
and query.dtype is torch.bfloat16
and (verified or _SANA_VIDEO_ROPE.can_attempt_once())
):
try:
@@ -11,7 +11,6 @@ import torch.nn as nn
from sglang.kernels.ops.diffusion import (
BitExactFusionGate,
can_use_fused_temb_table_slices,
can_use_linear_gelu,
fused_gelu_active,
fused_linear_gelu_tanh,
@@ -410,7 +409,8 @@ def _wan_temb_table_slices(
verified = _WAN_TEMB_SLICES.verified
if (
not _WAN_TEMB_SLICES.disabled
and can_use_fused_temb_table_slices(table, temb)
and _is_cuda
and temb.is_cuda
and (verified or _WAN_TEMB_SLICES.can_attempt_once())
):
try:
@@ -576,17 +576,20 @@ def materialize_overlay_model(
overlay_repo_id = str(overlay_spec["overlay_repo_id"])
overlay_revision = str(overlay_spec.get("overlay_revision", "main"))
overlay_fingerprint = _compute_overlay_fingerprint(overlay_dir)
source_revision = source_revision or _overlay_source_revision(
overlay_spec, manifest
)
cache_identity = {
"source_model_id": source_model_id,
"overlay_repo_id": overlay_repo_id,
"overlay_revision": overlay_revision,
"materializer_version": materializer_version,
"overlay_fingerprint": overlay_fingerprint,
}
if source_revision is not None:
cache_identity["source_revision"] = source_revision
cache_key = hashlib.sha256(
json.dumps(
{
"source_model_id": source_model_id,
"overlay_repo_id": overlay_repo_id,
"overlay_revision": overlay_revision,
"materializer_version": materializer_version,
"overlay_fingerprint": overlay_fingerprint,
},
sort_keys=True,
).encode("utf-8")
json.dumps(cache_identity, sort_keys=True).encode("utf-8")
).hexdigest()[:16]
cache_root = os.path.join(get_diffusion_cache_root(), "materialized_models")
_ensure_dir(cache_root)
@@ -449,8 +449,19 @@ def _run_file_suite(args, target_dir: Path) -> int:
print(f"No valid test files found for suite '{args.suite}'.")
return 1 if args.suite in STRICT_SUITES else 0
my_files = partition_items_by_index(
suite_files_abs, args.partition_id, args.total_partitions
)
print(
f"Suite: {args.suite} | Partition: {args.partition_id}/{args.total_partitions} "
f"| Files: {len(my_files)}/{len(suite_files_abs)}",
flush=True,
)
if not my_files:
return 0
exit_code, _, _ = run_pytest(
suite_files_abs,
my_files,
filter_expr=args.filter,
junit_xml_path=None,
)
@@ -356,10 +356,13 @@ class _Stage:
["--component-residency", "all=component-offload"],
"component-offload",
),
(
pytest.param(
"text_encoder_2",
["--component-residency", "text_encoder=snapshot-offload"],
"snapshot-offload",
marks=pytest.mark.skipif(
not current_platform.is_cuda(), reason="snapshot-offload requires CUDA"
),
),
("text_encoder_2", ["--text-encoder-cpu-offload", "true"], "component-offload"),
("text_encoder_2", ["--text-encoder-cpu-offload", "false"], "resident"),
@@ -24,6 +24,7 @@ from sglang.multimodal_gen.test.server.testcase_configs import (
@pytest.fixture
def validator(monkeypatch):
monkeypatch.setenv("SGLANG_GEN_BASELINE", "0")
monkeypatch.setattr(utils.current_platform, "is_hip", lambda: False)
scenario = ScenarioConfig.from_dict(
{
"stages_ms": {},
@@ -1,6 +1,11 @@
"""Unit tests for diffusion model-overlay cache paths and materialized trees."""
import json
import os
from pathlib import Path
import pytest
from filelock import FileLock
from sglang.multimodal_gen.runtime.utils import model_overlay
from sglang.multimodal_gen.runtime.utils.model_overlay import (
@@ -65,6 +70,80 @@ def test_large_files_are_still_shared_with_the_cache(tmp_path, monkeypatch):
assert os.path.samefile(out / "model.safetensors", weights)
@pytest.mark.parametrize("repair_cache", [False, True])
def test_overlay_uses_pinned_source_weights(tmp_path, monkeypatch, repair_cache):
overlay = tmp_path / "overlay"
(overlay / "_overlay").mkdir(parents=True)
(overlay / "_overlay" / "overlay_manifest.json").write_text(
json.dumps(
{
"required_source_files": ["model.safetensors"],
"file_mappings": [
{"src": "model.safetensors", "dst": "transformer/model.safetensors"}
],
}
)
)
(overlay / "model_index.json").write_text(
json.dumps(
{
"_class_name": "TestPipeline",
"_diffusers_version": "0.35.0",
"transformer": ["diffusers", "Transformer"],
}
)
)
for revision in ("main", "original", "newer"):
source = tmp_path / revision
source.mkdir()
if revision != "original" or not repair_cache:
(source / "model.safetensors").write_text(revision)
spec = {"overlay_repo_id": "test/overlay", "source_revision": "original"}
monkeypatch.setattr(
model_overlay, "resolve_model_overlay_target", lambda _: ("test/model", spec)
)
monkeypatch.setattr(
model_overlay, "download_overlay_metadata", lambda *a, **kw: str(overlay)
)
monkeypatch.setattr(
model_overlay, "get_diffusion_cache_root", lambda: str(tmp_path / "cache")
)
monkeypatch.setattr(
model_overlay, "get_lock", lambda _: FileLock(tmp_path / "overlay.lock")
)
def cached_snapshot(*args, revision=None, **kwargs):
return str(tmp_path / (revision or "main"))
def download_snapshot(*args, revision=None, **kwargs):
source = Path(cached_snapshot(revision=revision))
(source / "model.safetensors").write_text(revision or "main")
return str(source)
def resolve():
return model_overlay.maybe_resolve_overlay_model_path(
"test/model",
local_dir=None,
download=True,
allow_patterns=None,
snapshot_download_fn=download_snapshot,
hf_hub_download_fn=None,
verify_diffusers_model_complete_fn=lambda path: os.path.isfile(
os.path.join(path, "model_index.json")
),
base_model_download_fn=cached_snapshot,
)
original = Path(resolve())
assert (original / "transformer/model.safetensors").read_text() == "original"
# A new source revision must not reuse weights materialized from the old one.
spec["source_revision"] = "newer"
newer = Path(resolve())
assert (newer / "transformer/model.safetensors").read_text() == "newer"
assert original != newer
def test_registry_source_revision_survives_upstream_removing_required_file(
tmp_path, monkeypatch
):
@@ -38,6 +38,7 @@ from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.qwen_image21 import (
QwenImage21DenoisingStage,
)
from sglang.multimodal_gen.runtime.platforms import current_platform
from sglang.multimodal_gen.runtime.server_args import (
ServerArgs,
set_global_server_args,
@@ -46,7 +47,10 @@ from sglang.multimodal_gen.test.single_test_file.component_accuracy.utils import
ensure_distributed_env_defaults,
)
pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")
pytestmark = pytest.mark.skipif(
not (torch.cuda.is_available() and current_platform.is_cuda()),
reason="requires NVIDIA CUDA fusions",
)
@pytest.fixture(scope="module")
@@ -2,6 +2,7 @@ from dataclasses import replace
import pytest
from sglang.multimodal_gen.runtime.platforms import current_platform
from sglang.multimodal_gen.test.server.test_server_utils import PerformanceValidator
from sglang.multimodal_gen.test.server.testcase_configs import (
BaselineConfig,
@@ -69,6 +70,7 @@ def test_runner_override_preserves_other_metrics(monkeypatch, runner, flux, qwen
],
)
def test_runner_baseline_enforces_e2e_boundary(monkeypatch, runner):
monkeypatch.setattr(current_platform, "is_hip", lambda: False)
monkeypatch.setenv("RUNNER_NAME", runner)
monkeypatch.setenv("SGLANG_GEN_BASELINE", "0")
monkeypatch.delenv("SGLANG_E2E_TOLERANCE", raising=False)
@@ -124,6 +124,7 @@ def harness(monkeypatch):
monkeypatch.setenv("SGLANG_GEN_BASELINE", "0")
monkeypatch.setenv("SGLANG_SKIP_CONSISTENCY", "0")
monkeypatch.setattr(test_server_common.current_platform, "is_cuda", lambda: True)
monkeypatch.setattr(test_server_common.current_platform, "is_hip", lambda: False)
scenario = ScenarioConfig(
stages_ms={"DenoisingStage": 10},
denoise_step_ms={0: 5, 1: 5},
@@ -81,6 +81,34 @@ def test_suite_is_fully_scheduled_for_any_shard_count(suite, total_partitions):
assert sorted(scheduled_standalone_files) == sorted(expected_standalone_files)
@pytest.mark.parametrize("total_partitions", [1, 4, 8])
def test_file_suite_runs_each_file_once(monkeypatch, tmp_path, total_partitions):
files = [f"test_{i}.py" for i in range(5)]
for filename in files:
(tmp_path / filename).touch()
monkeypatch.setitem(run_suite.FILE_SUITES, "unit", files)
executed = []
def run_files(assigned, filter_expr, junit_xml_path):
assert assigned
assert filter_expr == "not ltx2_vae_channels_last"
executed.extend(assigned)
return 1, [], {}
monkeypatch.setattr(run_suite, "run_pytest", run_files)
for partition_id in range(total_partitions):
args = SimpleNamespace(
suite="unit",
partition_id=partition_id,
total_partitions=total_partitions,
filter="not ltx2_vae_channels_last",
)
result = run_suite._run_file_suite(args, tmp_path)
assert result == (1 if partition_id < len(files) else 0)
assert sorted(executed) == sorted(str(tmp_path / filename) for filename in files)
def test_failing_cases_do_not_skip_the_shards_standalone_files(monkeypatch, tmp_path):
"""Standalone files used to own a shard, so cases could not block them."""
standalone_rel = "../single_test_file/test_disagg_server.py"
@@ -6,10 +6,14 @@ from sglang.multimodal_gen.runtime.models.dits.wanvideo import (
_eager_temb_table_slices,
_wan_temb_table_slices,
)
from sglang.multimodal_gen.runtime.platforms import current_platform
class TestWanTembTableSlices(unittest.TestCase):
@unittest.skipUnless(torch.cuda.is_available(), "requires CUDA")
@unittest.skipUnless(
torch.cuda.is_available() and current_platform.is_cuda(),
"requires NVIDIA CUDA fusion",
)
def test_fused_matches_eager_and_is_contiguous(self):
torch.manual_seed(0)
for batch, seq, hidden in [(1, 517, 3072), (2, 64, 1536)]: