[diffusion] move kernel validation into launchers and simplify dispatch (#42391)
This commit is contained in:
@@ -52,25 +52,33 @@ __global__ void gelu_tanh_cat_kernel(
|
||||
template <typename T>
|
||||
void gelu_tanh_cat(tvm::ffi::TensorView attn, tvm::ffi::TensorView mlp, tvm::ffi::TensorView output) {
|
||||
using namespace host;
|
||||
auto rows = SymbolicSize{"rows"};
|
||||
auto attn_width = SymbolicSize{"attn_width"};
|
||||
auto mlp_width = SymbolicSize{"mlp_width"};
|
||||
CHECK_HOST(attn.ndim() >= 2) << "gelu_tanh_cat: expected at least two dimensions";
|
||||
auto dtype = SymbolicDType{};
|
||||
dtype.set_options<T>();
|
||||
auto device_ = SymbolicDevice{};
|
||||
device_.set_options<kDLCUDA>();
|
||||
TensorMatcher({rows, attn_width}).with_dtype<T>().with_device(device_).verify(attn);
|
||||
TensorMatcher({rows, mlp_width}).with_dtype<T>().with_device(device_).verify(mlp);
|
||||
const int64_t a = attn_width.unwrap();
|
||||
const int64_t m = mlp_width.unwrap();
|
||||
TensorMatcher({rows, a + m}).with_dtype<T>().with_device(device_).verify(output);
|
||||
for (const auto tensor : {attn, mlp, output}) {
|
||||
dtype.verify(tensor.dtype());
|
||||
device_.verify(tensor.device());
|
||||
CHECK_HOST(tensor.ndim() == attn.ndim() && tensor.is_contiguous())
|
||||
<< "gelu_tanh_cat: expected contiguous tensors of equal rank";
|
||||
for (int64_t dim = 0; dim < attn.ndim() - 1; ++dim) {
|
||||
CHECK_HOST(tensor.size(dim) == attn.size(dim)) << "gelu_tanh_cat: leading dimensions must match";
|
||||
}
|
||||
}
|
||||
const int64_t a = attn.size(-1);
|
||||
const int64_t m = mlp.size(-1);
|
||||
CHECK_HOST(output.size(-1) == a + m) << "gelu_tanh_cat: incorrect output width";
|
||||
constexpr int kVecN = 16 / sizeof(T);
|
||||
CHECK_HOST(a > 0 && m > 0 && a % kVecN == 0 && m % kVecN == 0)
|
||||
<< "gelu_tanh_cat: both widths must be positive multiples of " << kVecN;
|
||||
CHECK_HOST(rows.unwrap() > 0) << "gelu_tanh_cat: rows must be positive";
|
||||
const int64_t rows = attn.numel() / a;
|
||||
CHECK_HOST(rows > 0) << "gelu_tanh_cat: rows must be positive";
|
||||
CHECK_HOST(
|
||||
reinterpret_cast<uintptr_t>(attn.data_ptr()) % 16 == 0 && reinterpret_cast<uintptr_t>(mlp.data_ptr()) % 16 == 0 &&
|
||||
reinterpret_cast<uintptr_t>(output.data_ptr()) % 16 == 0)
|
||||
<< "gelu_tanh_cat: tensors must be 16-byte aligned";
|
||||
const int64_t n = rows.unwrap() * ((a + m) / kVecN);
|
||||
const int64_t n = rows * ((a + m) / kVecN);
|
||||
CHECK_HOST(n <= std::numeric_limits<int32_t>::max()) << "gelu_tanh_cat: tensor is too large";
|
||||
constexpr int kBlockSize = 256;
|
||||
const auto kernel = gelu_tanh_cat_kernel<T, kVecN>;
|
||||
|
||||
@@ -19,7 +19,7 @@ namespace sglang {
|
||||
* multiply and add/subtract operations preserve the separate eager FP32
|
||||
* intermediates before the result is rounded back to fp16/bf16.
|
||||
*
|
||||
* \tparam T Activation type: fp16_t or bf16_t
|
||||
* \tparam T Activation type: fp16_t, bf16_t or fp32_t
|
||||
* \param q Normalized query tensor, contiguous [tokens, heads, head_dim]
|
||||
* \param k Normalized key tensor, contiguous [tokens, heads, head_dim]
|
||||
* \param freqs Transposed Helios frequency tensor, contiguous
|
||||
@@ -38,7 +38,7 @@ __global__ void helios_qk_rope_kernel(
|
||||
uint32_t pairs_per_head,
|
||||
uint32_t num_heads,
|
||||
uint32_t freq_stride) {
|
||||
static_assert(std::is_same_v<T, fp16_t> || std::is_same_v<T, bf16_t>);
|
||||
static_assert(std::is_same_v<T, fp16_t> || std::is_same_v<T, bf16_t> || std::is_same_v<T, fp32_t>);
|
||||
using Packed = packed_t<T>;
|
||||
|
||||
auto* q_pairs = reinterpret_cast<Packed*>(q);
|
||||
@@ -79,10 +79,18 @@ struct HeliosQKRoPEKernel {
|
||||
auto device = SymbolicDevice{};
|
||||
device.set_options<kDLCUDA>();
|
||||
|
||||
TensorMatcher({N, H, D}).with_dtype<DType>().with_device(device).verify(q).verify(k);
|
||||
TensorMatcher({N, F}).with_dtype<fp32_t>().with_device(device).verify(freqs);
|
||||
int64_t batch = 1;
|
||||
if (q.ndim() == 4) {
|
||||
auto B = SymbolicSize{"batch"};
|
||||
TensorMatcher({B, N, H, D}).with_dtype<DType>().with_device(device).verify(q).verify(k);
|
||||
TensorMatcher({B, N, F}).with_dtype<fp32_t>().with_device(device).verify(freqs);
|
||||
batch = B.unwrap();
|
||||
} else {
|
||||
TensorMatcher({N, H, D}).with_dtype<DType>().with_device(device).verify(q).verify(k);
|
||||
TensorMatcher({N, F}).with_dtype<fp32_t>().with_device(device).verify(freqs);
|
||||
}
|
||||
|
||||
const int64_t tokens = N.unwrap();
|
||||
const int64_t tokens = batch * N.unwrap();
|
||||
const int64_t heads = H.unwrap();
|
||||
const int64_t head_dim = D.unwrap();
|
||||
const int64_t freq_dim = F.unwrap();
|
||||
|
||||
@@ -91,27 +91,27 @@ struct InterleavedRopeFP64Kernel {
|
||||
tvm::ffi::TensorView q,
|
||||
tvm::ffi::TensorView k,
|
||||
tvm::ffi::TensorView cos,
|
||||
tvm::ffi::TensorView sin,
|
||||
int64_t batch_size,
|
||||
int64_t seq_len,
|
||||
int64_t num_heads,
|
||||
int64_t head_dim) {
|
||||
tvm::ffi::TensorView sin) {
|
||||
using namespace host;
|
||||
using Pair = device::AlignedVector<T, 2>;
|
||||
|
||||
auto N = SymbolicSize{"activation_elements"};
|
||||
auto R = SymbolicSize{"table_elements"};
|
||||
auto B = SymbolicSize{"batch"};
|
||||
auto S = SymbolicSize{"sequence"};
|
||||
auto H = SymbolicSize{"heads"};
|
||||
auto D = SymbolicSize{"head_dim"};
|
||||
auto device = SymbolicDevice{};
|
||||
device.set_options<kDLCUDA>();
|
||||
TensorMatcher({N}).with_dtype<T>().with_device(device).verify(q_out).verify(k_out).verify(q).verify(k);
|
||||
TensorMatcher({R}).with_dtype<double>().with_device(device).verify(cos).verify(sin);
|
||||
TensorMatcher({B, S, H, D}).with_dtype<T>().with_device(device).verify(q_out).verify(k_out).verify(q).verify(k);
|
||||
TensorMatcher({1, S, 1, D}).with_dtype<double>().with_device(device).verify(cos).verify(sin);
|
||||
|
||||
const int64_t batch_size = B.unwrap();
|
||||
const int64_t seq_len = S.unwrap();
|
||||
const int64_t num_heads = H.unwrap();
|
||||
const int64_t head_dim = D.unwrap();
|
||||
|
||||
CHECK_HOST(batch_size > 0 && seq_len > 0 && num_heads > 0 && head_dim > 0)
|
||||
<< "interleaved_rope_fp64 dimensions must be positive";
|
||||
CHECK_HOST(head_dim % 2 == 0) << "interleaved_rope_fp64 head_dim must be even";
|
||||
CHECK_HOST(N.unwrap() == batch_size * seq_len * num_heads * head_dim)
|
||||
<< "interleaved_rope_fp64 activation shape does not match dimensions";
|
||||
CHECK_HOST(R.unwrap() == seq_len * head_dim) << "interleaved_rope_fp64 table shape does not match dimensions";
|
||||
CHECK_HOST(
|
||||
q_out.data_ptr() != k_out.data_ptr() && q_out.data_ptr() != q.data_ptr() && q_out.data_ptr() != k.data_ptr() &&
|
||||
k_out.data_ptr() != q.data_ptr() && k_out.data_ptr() != k.data_ptr())
|
||||
@@ -123,7 +123,7 @@ struct InterleavedRopeFP64Kernel {
|
||||
reinterpret_cast<uintptr_t>(k.data_ptr()) % alignof(Pair) == 0)
|
||||
<< "interleaved_rope_fp64 activations must be aligned to rotation pairs";
|
||||
|
||||
const int64_t num_pairs = N.unwrap() / 2;
|
||||
const int64_t num_pairs = q.numel() / 2;
|
||||
const int64_t pairs_per_head = head_dim / 2;
|
||||
const auto blocks =
|
||||
static_cast<uint32_t>(std::min<int64_t>(div_ceil(num_pairs, static_cast<int64_t>(kBlockSize)), kMaxGrid));
|
||||
|
||||
@@ -122,27 +122,32 @@ struct LTX25DecoderRopeKernel {
|
||||
tvm::ffi::TensorView sin_h,
|
||||
tvm::ffi::TensorView cos_w,
|
||||
tvm::ffi::TensorView sin_w,
|
||||
int64_t batch_size,
|
||||
int64_t num_frames,
|
||||
int64_t height,
|
||||
int64_t width,
|
||||
int64_t num_heads,
|
||||
int64_t head_dim,
|
||||
int64_t dim_t,
|
||||
int64_t dim_h) {
|
||||
using namespace host;
|
||||
using Pair = device::AlignedVector<T, 2>;
|
||||
|
||||
auto N = SymbolicSize{"activation_elements"};
|
||||
auto RT = SymbolicSize{"temporal_table_elements"};
|
||||
auto RH = SymbolicSize{"height_table_elements"};
|
||||
auto RW = SymbolicSize{"width_table_elements"};
|
||||
auto B = SymbolicSize{"batch"};
|
||||
auto F = SymbolicSize{"frames"};
|
||||
auto H = SymbolicSize{"height"};
|
||||
auto W = SymbolicSize{"width"};
|
||||
auto A = SymbolicSize{"heads"};
|
||||
auto D = SymbolicSize{"head_dim"};
|
||||
auto device = SymbolicDevice{};
|
||||
device.set_options<kDLCUDA>();
|
||||
TensorMatcher({N}).with_dtype<T>().with_device(device).verify(q_out).verify(k_out).verify(q).verify(k);
|
||||
TensorMatcher({RT}).with_dtype<float>().with_device(device).verify(cos_t).verify(sin_t);
|
||||
TensorMatcher({RH}).with_dtype<float>().with_device(device).verify(cos_h).verify(sin_h);
|
||||
TensorMatcher({RW}).with_dtype<float>().with_device(device).verify(cos_w).verify(sin_w);
|
||||
TensorMatcher({B, F, H, W, A, D})
|
||||
.with_dtype<T>()
|
||||
.with_device(device)
|
||||
.verify(q_out)
|
||||
.verify(k_out)
|
||||
.verify(q)
|
||||
.verify(k);
|
||||
const int64_t batch_size = B.unwrap();
|
||||
const int64_t num_frames = F.unwrap();
|
||||
const int64_t height = H.unwrap();
|
||||
const int64_t width = W.unwrap();
|
||||
const int64_t num_heads = A.unwrap();
|
||||
const int64_t head_dim = D.unwrap();
|
||||
|
||||
CHECK_HOST(batch_size > 0 && num_frames > 0 && height > 0 && width > 0 && num_heads > 0 && head_dim > 0)
|
||||
<< "ltx25_decoder_rope dimensions must be positive";
|
||||
@@ -150,11 +155,9 @@ struct LTX25DecoderRopeKernel {
|
||||
<< "ltx25_decoder_rope dimensions must split into positive rotation pairs";
|
||||
const int64_t dim_w = head_dim - dim_t - dim_h;
|
||||
CHECK_HOST(dim_w > 0 && dim_w % 2 == 0) << "ltx25_decoder_rope width dimension must be positive and even";
|
||||
CHECK_HOST(N.unwrap() == batch_size * num_frames * height * width * num_heads * head_dim)
|
||||
<< "ltx25_decoder_rope activation shape does not match dimensions";
|
||||
CHECK_HOST(RT.unwrap() == num_frames * dim_t / 2) << "ltx25_decoder_rope temporal table shape mismatch";
|
||||
CHECK_HOST(RH.unwrap() == height * dim_h / 2) << "ltx25_decoder_rope height table shape mismatch";
|
||||
CHECK_HOST(RW.unwrap() == width * dim_w / 2) << "ltx25_decoder_rope width table shape mismatch";
|
||||
TensorMatcher({F, dim_t / 2}).with_dtype<float>().with_device(device).verify(cos_t).verify(sin_t);
|
||||
TensorMatcher({H, dim_h / 2}).with_dtype<float>().with_device(device).verify(cos_h).verify(sin_h);
|
||||
TensorMatcher({W, dim_w / 2}).with_dtype<float>().with_device(device).verify(cos_w).verify(sin_w);
|
||||
CHECK_HOST(
|
||||
q_out.data_ptr() != k_out.data_ptr() && q_out.data_ptr() != q.data_ptr() && q_out.data_ptr() != k.data_ptr() &&
|
||||
k_out.data_ptr() != q.data_ptr() && k_out.data_ptr() != k.data_ptr())
|
||||
@@ -166,7 +169,7 @@ struct LTX25DecoderRopeKernel {
|
||||
reinterpret_cast<uintptr_t>(k.data_ptr()) % alignof(Pair) == 0)
|
||||
<< "ltx25_decoder_rope activations must be aligned to rotation pairs";
|
||||
|
||||
const int64_t num_pairs = N.unwrap() / 2;
|
||||
const int64_t num_pairs = q.numel() / 2;
|
||||
const int64_t pairs_per_head = head_dim / 2;
|
||||
const auto blocks =
|
||||
static_cast<uint32_t>(std::min<int64_t>(div_ceil(num_pairs, static_cast<int64_t>(kBlockSize)), kMaxGrid));
|
||||
|
||||
@@ -211,7 +211,8 @@ __global__ void kernel(const Params __grid_constant__ params) {
|
||||
|
||||
template <bool kFuseRealSin, bool kCopyV>
|
||||
struct Kernel {
|
||||
/// \brief Single tensor, out of place: `out = rope(rmsnorm(x))` for a contiguous [B, S, H, 128] bf16 tensor.
|
||||
/// \brief Single tensor, out of place: `out = rope(rmsnorm(x))` for a [B, S, H, 128] bf16 tensor with packed head
|
||||
/// rows.
|
||||
static void
|
||||
run(tvm::ffi::TensorView x,
|
||||
tvm::ffi::TensorView out,
|
||||
@@ -224,19 +225,31 @@ struct Kernel {
|
||||
auto H = SymbolicSize{"heads"};
|
||||
auto device = SymbolicDevice{};
|
||||
device.set_options<kDLCUDA>();
|
||||
auto Bx = SymbolicSize{"input_batch_stride"};
|
||||
auto Tx = SymbolicSize{"input_token_stride"};
|
||||
TensorMatcher({B, S, H, kHeadDim})
|
||||
.with_strides({Bx, Tx, kHeadDim, 1})
|
||||
.with_dtype<bf16_t>()
|
||||
.with_device(device)
|
||||
.ensure_alignment(kVecAlignBytes)
|
||||
.verify(x);
|
||||
TensorMatcher({B, S, H, kHeadDim})
|
||||
.with_dtype<bf16_t>()
|
||||
.with_device(device)
|
||||
.ensure_alignment(kVecAlignBytes)
|
||||
.verify(x)
|
||||
.verify(out);
|
||||
TensorMatcher({kHeadDim}).with_dtype<bf16_t>().with_device(device).verify(weight);
|
||||
TensorMatcher({S, kHeadDim}).with_dtype<fp32_t>().with_device(device).ensure_alignment(16).verify(rope);
|
||||
TensorMatcher({S, kHeadDim / 2, 2}).with_dtype<fp32_t>().with_device(device).verify(rope);
|
||||
// float4 loads span two complex pairs; only the token rows need 16-byte alignment.
|
||||
CHECK_HOST(reinterpret_cast<uintptr_t>(rope.data_ptr()) % 16 == 0) << "RoPE cache must be 16-byte aligned";
|
||||
CHECK_HOST(
|
||||
Bx.unwrap() % (kVecAlignBytes / sizeof(bf16_t)) == 0 && Tx.unwrap() % (kVecAlignBytes / sizeof(bf16_t)) == 0)
|
||||
<< "input batch/token strides must preserve vector alignment";
|
||||
const int64_t batch = B.unwrap(), seq = S.unwrap(), heads = H.unwrap();
|
||||
if (batch == 0 || seq == 0 || heads == 0) return;
|
||||
const int64_t token_stride = heads * kHeadDim;
|
||||
Params params{};
|
||||
params.q_src = Rows{static_cast<const bf16_t*>(x.data_ptr()), seq * token_stride, token_stride};
|
||||
params.q_src = Rows{static_cast<const bf16_t*>(x.data_ptr()), Bx.unwrap(), Tx.unwrap()};
|
||||
params.q_dst = MutableRows{static_cast<bf16_t*>(out.data_ptr()), seq * token_stride, token_stride};
|
||||
params.q_weight = static_cast<const bf16_t*>(weight.data_ptr());
|
||||
params.k_weight = params.q_weight;
|
||||
@@ -301,7 +314,9 @@ struct Kernel {
|
||||
.verify(k_prefix)
|
||||
.verify(v_prefix);
|
||||
TensorMatcher({kHeadDim}).with_dtype<bf16_t>().with_device(device).verify(q_weight).verify(k_weight);
|
||||
TensorMatcher({S, kHeadDim}).with_dtype<fp32_t>().with_device(device).ensure_alignment(16).verify(rope);
|
||||
TensorMatcher({S, kHeadDim / 2, 2}).with_dtype<fp32_t>().with_device(device).verify(rope);
|
||||
// float4 loads span two complex pairs; only the token rows need 16-byte alignment.
|
||||
CHECK_HOST(reinterpret_cast<uintptr_t>(rope.data_ptr()) % 16 == 0) << "RoPE cache must be 16-byte aligned";
|
||||
const int64_t batch = B.unwrap(), seq = S.unwrap(), heads = H.unwrap(), prefix = P.unwrap();
|
||||
CHECK_HOST(PS.unwrap() == prefix + seq) << "k_out/v_out must hold prefix + seq tokens";
|
||||
if (batch == 0 || seq == 0 || heads == 0) return;
|
||||
|
||||
@@ -185,11 +185,11 @@ SGL_DEVICE void step(float2 (&t)[8][4], Smem& sm, int kt, int ti, int tj, int i0
|
||||
/**
|
||||
* \brief transition = diag(alpha) (I + A)^-1, injection = B (I + A)^-1 for a batch of 128x128 SPD A.
|
||||
*
|
||||
* \param A [N, 128, 128] fp32, symmetric positive semi-definite (I + A is inverted)
|
||||
* \param B [N, 128, 128] fp32
|
||||
* \param alpha [N, 128] fp32 row scales of the transition
|
||||
* \param transition [N, 128, 128] fp32 output
|
||||
* \param injection [N, 128, 128] fp32 output
|
||||
* \param A [..., 128, 128] fp32, symmetric positive semi-definite (I + A is inverted)
|
||||
* \param B [..., 128, 128] fp32
|
||||
* \param alpha [..., 128] fp32 row scales of the transition
|
||||
* \param transition [..., 128, 128] fp32 output
|
||||
* \param injection [..., 128, 128] fp32 output
|
||||
*/
|
||||
__global__ void __launch_bounds__(kBlockSize, kMinBlocksPerSm) vdn_delta_factors_kernel(
|
||||
const float* __restrict__ A,
|
||||
@@ -305,11 +305,11 @@ struct VdnDeltaFactorsKernel {
|
||||
/**
|
||||
* \brief Validate the tensors and launch one CTA per matrix.
|
||||
*
|
||||
* \param transition [N, 128, 128] fp32 output, diag(alpha) (I + A)^-1
|
||||
* \param injection [N, 128, 128] fp32 output, B (I + A)^-1
|
||||
* \param A [N, 128, 128] fp32 SPD statistics (I + A is inverted)
|
||||
* \param B [N, 128, 128] fp32
|
||||
* \param alpha [N, 128] fp32
|
||||
* \param transition [..., 128, 128] fp32 output, diag(alpha) (I + A)^-1
|
||||
* \param injection [..., 128, 128] fp32 output, B (I + A)^-1
|
||||
* \param A [..., 128, 128] fp32 SPD statistics (I + A is inverted)
|
||||
* \param B [..., 128, 128] fp32
|
||||
* \param alpha [..., 128] fp32
|
||||
*/
|
||||
static void
|
||||
run(tvm::ffi::TensorView transition,
|
||||
@@ -318,18 +318,28 @@ struct VdnDeltaFactorsKernel {
|
||||
tvm::ffi::TensorView B,
|
||||
tvm::ffi::TensorView alpha) {
|
||||
using namespace host;
|
||||
auto N = SymbolicSize{"num_matrices"};
|
||||
CHECK_HOST(A.ndim() >= 2 && A.size(-2) == kDim && A.size(-1) == kDim)
|
||||
<< "vdn_delta_factors expects [..., 128, 128] matrices";
|
||||
auto dtype = SymbolicDType{};
|
||||
dtype.set_options<fp32_t>();
|
||||
auto device = SymbolicDevice{};
|
||||
device.set_options<kDLCUDA>();
|
||||
TensorMatcher({N, kDim, kDim})
|
||||
.with_dtype<fp32_t>()
|
||||
.with_device(device)
|
||||
.verify(transition)
|
||||
.verify(injection)
|
||||
.verify(A)
|
||||
.verify(B);
|
||||
TensorMatcher({N, kDim}).with_dtype<fp32_t>().with_device(device).verify(alpha);
|
||||
const int64_t num = N.unwrap();
|
||||
for (const auto tensor : {A, B, transition, injection, alpha}) {
|
||||
dtype.verify(tensor.dtype());
|
||||
device.verify(tensor.device());
|
||||
CHECK_HOST(tensor.is_contiguous()) << "vdn_delta_factors expects contiguous tensors";
|
||||
}
|
||||
for (const auto tensor : {B, transition, injection}) {
|
||||
CHECK_HOST(tensor.ndim() == A.ndim()) << "vdn_delta_factors matrix ranks must match";
|
||||
for (int64_t dim = 0; dim < A.ndim(); ++dim) {
|
||||
CHECK_HOST(tensor.size(dim) == A.size(dim)) << "vdn_delta_factors matrix shapes must match";
|
||||
}
|
||||
}
|
||||
CHECK_HOST(alpha.ndim() == A.ndim() - 1) << "alpha must have shape A.shape[:-1]";
|
||||
for (int64_t dim = 0; dim < alpha.ndim(); ++dim) {
|
||||
CHECK_HOST(alpha.size(dim) == A.size(dim)) << "alpha must have shape A.shape[:-1]";
|
||||
}
|
||||
const int64_t num = A.numel() / (kDim * kDim);
|
||||
if (num == 0) return;
|
||||
CHECK_HOST(
|
||||
transition.data_ptr() != A.data_ptr() && transition.data_ptr() != B.data_ptr() &&
|
||||
|
||||
@@ -17,6 +17,8 @@ _SUPPORTED_DTYPES = (torch.float16, torch.bfloat16, torch.float32)
|
||||
|
||||
@cache_once
|
||||
def _jit_causal_conv3d_cat_pad_module(dtype: torch.dtype) -> Module:
|
||||
if dtype not in _SUPPORTED_DTYPES:
|
||||
raise RuntimeError(f"unsupported dtype for causal Conv3D cat/pad: {dtype}")
|
||||
args = make_cpp_args(dtype)
|
||||
return load_jit(
|
||||
"diffusion_causal_conv3d_cat_pad",
|
||||
@@ -101,19 +103,7 @@ def fused_causal_conv3d_cat_pad_cuda(
|
||||
cache_x: torch.Tensor,
|
||||
padding: list[int] | tuple[int, ...],
|
||||
) -> torch.Tensor:
|
||||
if x.dtype not in _SUPPORTED_DTYPES:
|
||||
raise RuntimeError(f"unsupported dtype for causal Conv3D cat/pad: {x.dtype}")
|
||||
if not torch.compiler.is_compiling():
|
||||
if (
|
||||
not x.is_cuda
|
||||
or not cache_x.is_cuda
|
||||
or x.dim() != 5
|
||||
or cache_x.dim() != 5
|
||||
or not x.is_contiguous()
|
||||
or not cache_x.is_contiguous()
|
||||
or not can_use_fused_causal_conv3d_cat_pad_cuda(x, cache_x, padding)
|
||||
):
|
||||
raise RuntimeError("unsupported input for causal Conv3D cat/pad CUDA")
|
||||
"""Concatenate the temporal cache and pad; the C++ launcher validates inputs."""
|
||||
return _causal_conv3d_cat_pad_custom_op(x, cache_x, *padding)
|
||||
|
||||
|
||||
|
||||
@@ -26,8 +26,6 @@ constexpr uint32_t kBroadcastRowsPerBlock = 4;
|
||||
constexpr uint32_t kBroadcastColsPerBlock = 256;
|
||||
constexpr uint32_t kMaxGrid = 65535;
|
||||
constexpr uintptr_t kAlignment = 16;
|
||||
constexpr uint32_t kTransposeTile = 32;
|
||||
constexpr uint32_t kTransposeBlockSize = 256;
|
||||
|
||||
enum class GateMode : int { kFull, kBroadcastRow, kPerToken };
|
||||
|
||||
@@ -116,55 +114,6 @@ __global__ void residual_gate_add_scalar_kernel(
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fuse the residual-gate update when the residual/output use the common
|
||||
* ``[batch, hidden, tokens]`` backing layout exposed as ``[batch, tokens,
|
||||
* hidden]``, while update remains contiguous in the logical layout.
|
||||
*
|
||||
* Loading update in logical row-major order and consuming it transposed from
|
||||
* shared memory keeps both update reads and residual/output traffic coalesced.
|
||||
*/
|
||||
template <typename T>
|
||||
__global__ void residual_gate_add_transposed_kernel(
|
||||
T* __restrict__ out,
|
||||
const T* __restrict__ residual,
|
||||
const T* __restrict__ update,
|
||||
const T* __restrict__ gate,
|
||||
int64_t tokens,
|
||||
int64_t hidden_size) {
|
||||
__shared__ T update_tile[kTransposeTile][kTransposeTile + 1];
|
||||
|
||||
const int64_t batch = blockIdx.z;
|
||||
const int64_t token_base = static_cast<int64_t>(blockIdx.x) * kTransposeTile;
|
||||
const int64_t hidden_base = static_cast<int64_t>(blockIdx.y) * kTransposeTile;
|
||||
const int64_t batch_offset = batch * tokens * hidden_size;
|
||||
|
||||
#pragma unroll
|
||||
for (uint32_t item = threadIdx.x; item < kTransposeTile * kTransposeTile; item += kTransposeBlockSize) {
|
||||
const uint32_t token_in_tile = item / kTransposeTile;
|
||||
const uint32_t hidden_in_tile = item % kTransposeTile;
|
||||
const int64_t token = token_base + token_in_tile;
|
||||
const int64_t hidden = hidden_base + hidden_in_tile;
|
||||
if (token < tokens && hidden < hidden_size) {
|
||||
update_tile[token_in_tile][hidden_in_tile] = update[batch_offset + token * hidden_size + hidden];
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
#pragma unroll
|
||||
for (uint32_t item = threadIdx.x; item < kTransposeTile * kTransposeTile; item += kTransposeBlockSize) {
|
||||
const uint32_t hidden_in_tile = item / kTransposeTile;
|
||||
const uint32_t token_in_tile = item % kTransposeTile;
|
||||
const int64_t token = token_base + token_in_tile;
|
||||
const int64_t hidden = hidden_base + hidden_in_tile;
|
||||
if (token < tokens && hidden < hidden_size) {
|
||||
const int64_t transposed_offset = batch_offset + hidden * tokens + token;
|
||||
out[transposed_offset] = residual_gate_value(
|
||||
residual[transposed_offset], update_tile[token_in_tile][hidden_in_tile], SGLANG_LDG(gate + hidden));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
/**
|
||||
@@ -281,49 +230,6 @@ struct ResidualGateAddKernel {
|
||||
hidden_size);
|
||||
}
|
||||
}
|
||||
|
||||
static void run_transposed(
|
||||
tvm::ffi::TensorView out, tvm::ffi::TensorView residual, tvm::ffi::TensorView update, tvm::ffi::TensorView gate) {
|
||||
using namespace host;
|
||||
|
||||
auto B = SymbolicSize{"batch"};
|
||||
auto S = SymbolicSize{"tokens"};
|
||||
auto D = SymbolicSize{"hidden_size"};
|
||||
auto device = SymbolicDevice{};
|
||||
device.set_options<kDLCUDA>();
|
||||
TensorMatcher({B, S, D}).with_strides({-1, 1, -1}).with_dtype<T>().with_device(device).verify(out).verify(residual);
|
||||
TensorMatcher({B, S, D}).with_strides({-1, -1, 1}).with_dtype<T>().with_device(device).verify(update);
|
||||
TensorMatcher({1, 1, D}).with_strides({-1, -1, 1}).with_dtype<T>().with_device(device).verify(gate);
|
||||
|
||||
const int64_t batch = B.unwrap();
|
||||
const int64_t tokens = S.unwrap();
|
||||
const int64_t hidden_size = D.unwrap();
|
||||
CHECK_HOST(batch > 0 && tokens > 0 && hidden_size > 0) << "transposed residual-gate tensors must be non-empty";
|
||||
CHECK_HOST(
|
||||
batch <= kMaxGrid && div_ceil(tokens, int64_t{kTransposeTile}) <= kMaxGrid &&
|
||||
div_ceil(hidden_size, int64_t{kTransposeTile}) <= kMaxGrid)
|
||||
<< "transposed residual-gate grid exceeds CUDA limits";
|
||||
const int64_t batch_stride = tokens * hidden_size;
|
||||
CHECK_HOST(
|
||||
out.stride(0) == batch_stride && out.stride(2) == tokens && residual.stride(0) == batch_stride &&
|
||||
residual.stride(2) == tokens)
|
||||
<< "residual/output must use the transposed dense layout";
|
||||
CHECK_HOST(update.stride(0) == batch_stride && update.stride(1) == hidden_size) << "update must be contiguous";
|
||||
|
||||
auto* out_ptr = static_cast<T*>(out.data_ptr());
|
||||
const auto* residual_ptr = static_cast<const T*>(residual.data_ptr());
|
||||
const auto* update_ptr = static_cast<const T*>(update.data_ptr());
|
||||
const auto* gate_ptr = static_cast<const T*>(gate.data_ptr());
|
||||
CHECK_HOST(out_ptr != residual_ptr && out_ptr != update_ptr && out_ptr != gate_ptr)
|
||||
<< "output must not alias an input";
|
||||
|
||||
const dim3 grid(
|
||||
static_cast<uint32_t>(div_ceil(tokens, int64_t{kTransposeTile})),
|
||||
static_cast<uint32_t>(div_ceil(hidden_size, int64_t{kTransposeTile})),
|
||||
static_cast<uint32_t>(batch));
|
||||
LaunchKernel(grid, kTransposeBlockSize, device.unwrap())(
|
||||
residual_gate_add_transposed_kernel<T>, out_ptr, residual_ptr, update_ptr, gate_ptr, tokens, hidden_size);
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace residual_gate_add
|
||||
|
||||
@@ -402,33 +402,45 @@ def _qk_head_launch_config() -> tuple[int, int]:
|
||||
def _mod_row_stride(t: torch.Tensor, batch: int, hidden: int) -> int | None:
|
||||
# (batch, hidden) modulation rows, possibly strided views of a chunked
|
||||
# adaLN projection; the last dim must be packed.
|
||||
if t.dim() != 2 or t.shape != (batch, hidden) or t.stride(1) != 1:
|
||||
if t.shape != (batch, hidden) or t.stride(1) != 1:
|
||||
return None
|
||||
return t.stride(0) if batch > 1 else hidden
|
||||
|
||||
|
||||
def can_use_fused_layernorm_modulate(
|
||||
x: torch.Tensor, scale: torch.Tensor, shift: torch.Tensor | None
|
||||
) -> bool:
|
||||
def can_use_fused_layernorm_modulate(dtype: torch.dtype, hidden: int) -> bool:
|
||||
"""Select the BF16 vectorized LayerNorm reduction replicated below."""
|
||||
return (
|
||||
is_cuda() and dtype is torch.bfloat16 and 0 < hidden <= 8192 and hidden % 4 == 0
|
||||
)
|
||||
|
||||
|
||||
def _validate_layernorm_modulate(x, scale, shift) -> tuple[int, int, int, int]:
|
||||
shape = x.shape
|
||||
if not (
|
||||
_is_bf16_cuda(x)
|
||||
and x.dim() == 3
|
||||
and len(shape) == 3
|
||||
and x.numel() > 0
|
||||
and x.is_contiguous()
|
||||
and x.shape[-1] % 4 == 0
|
||||
and x.shape[-1] <= 8192
|
||||
and _is_bf16_cuda(scale)
|
||||
and scale.device == x.device
|
||||
and 0 < shape[-1] <= 8192
|
||||
and shape[-1] % 4 == 0
|
||||
):
|
||||
return False
|
||||
batch, _, hidden = x.shape
|
||||
q = _mod_row_stride(scale, batch, hidden)
|
||||
if shift is None:
|
||||
return q is not None
|
||||
if not _is_bf16_cuda(shift) or shift.device != x.device:
|
||||
return False
|
||||
v = _mod_row_stride(shift, batch, hidden)
|
||||
return q is not None and v is not None and q == v
|
||||
raise RuntimeError(
|
||||
"LayerNorm modulation expects contiguous BF16 CUDA [B, S, D], D divisible by 4 and <= 8192"
|
||||
)
|
||||
batch, seq_len, hidden = shape
|
||||
device = x.device
|
||||
stride = _mod_row_stride(scale, batch, hidden)
|
||||
if not (
|
||||
stride is not None and scale.dtype is torch.bfloat16 and scale.device == device
|
||||
):
|
||||
raise RuntimeError("scale must have packed BF16 [B, D] rows on x's device")
|
||||
if shift is not None and not (
|
||||
shift.dtype is torch.bfloat16
|
||||
and shift.device == device
|
||||
and _mod_row_stride(shift, batch, hidden) == stride
|
||||
):
|
||||
raise RuntimeError("shift must match scale's row layout and dtype/device")
|
||||
return batch, seq_len, hidden, stride
|
||||
|
||||
|
||||
def _fake_ln_modulate(
|
||||
@@ -510,11 +522,10 @@ def fused_layernorm_modulate_raw(
|
||||
of microseconds per call); use it on CPU-launch-bound eager hot paths
|
||||
(e.g. Sana), and the registered custom op under ``torch.compile``.
|
||||
"""
|
||||
batch, seq_len, hidden = x.shape
|
||||
batch, seq_len, hidden, stride = _validate_layernorm_modulate(x, scale, shift)
|
||||
n_rows = batch * seq_len
|
||||
rows = 2
|
||||
out = torch.empty_like(x)
|
||||
stride = _mod_row_stride(scale, batch, hidden)
|
||||
with torch.cuda.device(x.device):
|
||||
_layernorm_modulate_kernel[(triton.cdiv(n_rows, rows),)](
|
||||
out,
|
||||
@@ -558,11 +569,10 @@ def fused_layernorm_modulate_fp8_quant_raw(
|
||||
Unlike that two-op chain, this path does not materialize the intermediate
|
||||
BF16 activation because FLUX.2 feeds it directly into an FP8 projection.
|
||||
"""
|
||||
batch, seq_len, hidden = x.shape
|
||||
batch, seq_len, hidden, stride = _validate_layernorm_modulate(x, scale, shift)
|
||||
n_rows = batch * seq_len
|
||||
rows = 2
|
||||
out = torch.empty_like(x, dtype=fp8_dtype)
|
||||
stride = _mod_row_stride(scale, batch, hidden)
|
||||
with torch.cuda.device(x.device):
|
||||
_layernorm_modulate_kernel[(triton.cdiv(n_rows, rows),)](
|
||||
out,
|
||||
@@ -595,19 +605,13 @@ fused_layernorm_modulate = register_custom_op(
|
||||
)
|
||||
|
||||
|
||||
def can_use_fused_qk_head_layernorm(q: torch.Tensor, k: torch.Tensor) -> bool:
|
||||
head_dim = q.shape[-1] if q.dim() == 4 else 0
|
||||
def can_use_fused_qk_head_layernorm(dtype: torch.dtype, head_dim: int) -> bool:
|
||||
"""Select the BF16 per-head LayerNorm reduction."""
|
||||
return (
|
||||
_is_bf16_cuda(q)
|
||||
and _is_bf16_cuda(k)
|
||||
and q.device == k.device
|
||||
and q.dim() == 4
|
||||
and q.shape == k.shape
|
||||
and head_dim % 4 == 0
|
||||
is_cuda()
|
||||
and dtype is torch.bfloat16
|
||||
and 0 < head_dim <= 128
|
||||
and q.numel() > 0
|
||||
and q.is_contiguous()
|
||||
and k.is_contiguous()
|
||||
and head_dim % 4 == 0
|
||||
)
|
||||
|
||||
|
||||
@@ -627,7 +631,23 @@ def fused_qk_head_layernorm(
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Per-head ``nn.LayerNorm(dim_head)`` (no affine) over q and k in one
|
||||
launch, bit-exact vs the eager aten kernel."""
|
||||
head_dim = q.shape[-1]
|
||||
shape = q.shape
|
||||
if not (
|
||||
_is_bf16_cuda(q)
|
||||
and len(shape) == 4
|
||||
and q.numel() > 0
|
||||
and 0 < shape[-1] <= 128
|
||||
and shape[-1] % 4 == 0
|
||||
and k.dtype is torch.bfloat16
|
||||
and k.device == q.device
|
||||
and k.shape == shape
|
||||
and q.is_contiguous()
|
||||
and k.is_contiguous()
|
||||
):
|
||||
raise RuntimeError(
|
||||
"QK LayerNorm expects matching contiguous BF16 CUDA [B, S, H, D], D divisible by 4 and <= 128"
|
||||
)
|
||||
head_dim = shape[-1]
|
||||
n_rows = q.numel() // head_dim
|
||||
# Architecture sweeps at the production GLM shape select 32 rows / 1 warp
|
||||
# on B300 (SM103) and 8 rows / 4 warps on RTX 5090 (SM120). Preserve the
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
@@ -19,9 +18,6 @@ _SUPPORTED_DTYPES = (torch.float16, torch.bfloat16, torch.float32)
|
||||
_BIT_EXACT_DTYPES = (torch.float16, torch.bfloat16)
|
||||
_TRANSPOSE_TILE = 32
|
||||
_MAX_GRID_DIM = 65535
|
||||
_FAILED_RUNTIME_KEYS: set[tuple[int | None, torch.dtype]] = set()
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@cache_once
|
||||
@@ -38,10 +34,6 @@ def _jit_residual_gate_add_module(dtype: torch.dtype) -> Module:
|
||||
"residual_gate_add",
|
||||
f"residual_gate_add::ResidualGateAddKernel<{args}>::run",
|
||||
),
|
||||
(
|
||||
"residual_gate_add_transposed",
|
||||
f"residual_gate_add::ResidualGateAddKernel<{args}>::run_transposed",
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
@@ -67,9 +59,6 @@ def _residual_gate_add_cuda_impl(
|
||||
device=residual.device,
|
||||
)
|
||||
module = _jit_residual_gate_add_module(residual.dtype)
|
||||
if _is_transposed_dense_residual(residual, update, gate):
|
||||
module.residual_gate_add_transposed(out, residual, update, gate)
|
||||
return out
|
||||
gate_mode = _gate_mode(residual, gate)
|
||||
module.residual_gate_add(
|
||||
out.view(-1),
|
||||
@@ -96,120 +85,6 @@ def _round16_f32(x, IS_BF16: tl.constexpr):
|
||||
return bits.to(tl.float16, bitcast=True).to(tl.float32)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _store16(out_ptr, offs, value_f32, mask, IS_BF16: tl.constexpr):
|
||||
if IS_BF16:
|
||||
bits = tl.inline_asm_elementwise(
|
||||
"cvt.rn.bf16.f32 $0, $1;",
|
||||
"=h,r",
|
||||
[value_f32],
|
||||
dtype=tl.int16,
|
||||
is_pure=True,
|
||||
pack=1,
|
||||
)
|
||||
tl.store(out_ptr + offs, bits.to(tl.bfloat16, bitcast=True), mask=mask)
|
||||
else:
|
||||
bits = tl.inline_asm_elementwise(
|
||||
"cvt.rn.f16.f32 $0, $1;",
|
||||
"=h,r",
|
||||
[value_f32],
|
||||
dtype=tl.int16,
|
||||
is_pure=True,
|
||||
pack=1,
|
||||
)
|
||||
tl.store(out_ptr + offs, bits.to(tl.float16, bitcast=True), mask=mask)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _rga_flat(
|
||||
out,
|
||||
res,
|
||||
upd,
|
||||
gate,
|
||||
numel,
|
||||
hid,
|
||||
MODE: tl.constexpr,
|
||||
IS_BF16: tl.constexpr,
|
||||
IS_16: tl.constexpr,
|
||||
BLK: tl.constexpr,
|
||||
):
|
||||
pid = tl.program_id(0).to(tl.int64)
|
||||
offs = pid * BLK + tl.arange(0, BLK).to(tl.int64)
|
||||
mask = offs < numel
|
||||
rv = tl.load(res + offs, mask=mask)
|
||||
uv = tl.load(upd + offs, mask=mask)
|
||||
if MODE == 1:
|
||||
gv = tl.load(gate + (offs % hid), mask=mask)
|
||||
elif MODE == 2:
|
||||
gv = tl.load(gate + (offs // hid), mask=mask)
|
||||
else:
|
||||
gv = tl.load(gate + offs, mask=mask)
|
||||
if IS_16:
|
||||
p32 = uv.to(tl.float32) * gv.to(tl.float32)
|
||||
pf = _round16_f32(p32, IS_BF16)
|
||||
o32 = rv.to(tl.float32) + pf
|
||||
_store16(out, offs, o32, mask, IS_BF16)
|
||||
else:
|
||||
p32 = uv * gv
|
||||
o32 = rv + p32
|
||||
tl.store(out + offs, o32, mask=mask)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _rga_row(
|
||||
out,
|
||||
res,
|
||||
upd,
|
||||
gate,
|
||||
hid: tl.constexpr,
|
||||
IS_BF16: tl.constexpr,
|
||||
EVG: tl.constexpr,
|
||||
CG: tl.constexpr,
|
||||
BLK: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0).to(tl.int64)
|
||||
offs = tl.arange(0, BLK)
|
||||
mask = offs < hid
|
||||
if EVG:
|
||||
gv = tl.load(gate + offs, mask=mask, eviction_policy="evict_last")
|
||||
else:
|
||||
gv = tl.load(gate + offs, mask=mask)
|
||||
base = row * hid
|
||||
if CG:
|
||||
rv = tl.load(res + base + offs, mask=mask, cache_modifier=".cg")
|
||||
uv = tl.load(upd + base + offs, mask=mask, cache_modifier=".cg")
|
||||
else:
|
||||
rv = tl.load(res + base + offs, mask=mask, eviction_policy="evict_first")
|
||||
uv = tl.load(upd + base + offs, mask=mask, eviction_policy="evict_first")
|
||||
p32 = uv.to(tl.float32) * gv.to(tl.float32)
|
||||
pf = _round16_f32(p32, IS_BF16)
|
||||
o32 = rv.to(tl.float32) + pf
|
||||
_store16(out, base + offs, o32, mask, IS_BF16)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _rga_flat_m1(
|
||||
out,
|
||||
res,
|
||||
upd,
|
||||
gate,
|
||||
numel,
|
||||
hid: tl.constexpr,
|
||||
IS_BF16: tl.constexpr,
|
||||
BLK: tl.constexpr,
|
||||
):
|
||||
pid = tl.program_id(0).to(tl.int64)
|
||||
offs = pid * BLK + tl.arange(0, BLK).to(tl.int64)
|
||||
mask = offs < numel
|
||||
rv = tl.load(res + offs, mask=mask, eviction_policy="evict_first")
|
||||
uv = tl.load(upd + offs, mask=mask, eviction_policy="evict_first")
|
||||
gv = tl.load(gate + (offs % hid), mask=mask, eviction_policy="evict_last")
|
||||
p32 = uv.to(tl.float32) * gv.to(tl.float32)
|
||||
pf = _round16_f32(p32, IS_BF16)
|
||||
o32 = rv.to(tl.float32) + pf
|
||||
_store16(out, offs, o32, mask, IS_BF16)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _rga_transposed(
|
||||
out,
|
||||
@@ -268,146 +143,30 @@ def _rga_transposed(
|
||||
tl.store(out + base + roffs, rv + uv * gv[:, None], mask=m2)
|
||||
|
||||
|
||||
def _residual_gate_add_triton(residual, update, gate):
|
||||
# Both callers validate the inputs first; repeating the predicate here cost
|
||||
# ~2.7us of host time on every call.
|
||||
def _residual_gate_add_transposed(residual, update, gate):
|
||||
out = torch.empty_strided(
|
||||
residual.shape, residual.stride(), dtype=residual.dtype, device=residual.device
|
||||
)
|
||||
is_bf16 = residual.dtype == torch.bfloat16
|
||||
dtype16 = residual.dtype in (torch.float16, torch.bfloat16)
|
||||
|
||||
if _is_transposed_dense_residual(residual, update, gate):
|
||||
tokens = residual.shape[1]
|
||||
hid = residual.shape[2]
|
||||
batch = residual.shape[0]
|
||||
if tokens * hid * batch <= 65536:
|
||||
tile, warps = 16, 4
|
||||
elif tokens * hid * batch >= 1 << 20:
|
||||
tile, warps = 64, 8
|
||||
else:
|
||||
tile, warps = 32, 8
|
||||
grid = (triton.cdiv(tokens, tile), triton.cdiv(hid, tile), batch)
|
||||
_rga_transposed[grid](
|
||||
out,
|
||||
residual,
|
||||
update,
|
||||
gate,
|
||||
tokens,
|
||||
hid,
|
||||
is_bf16,
|
||||
dtype16,
|
||||
TILE=tile,
|
||||
num_warps=warps,
|
||||
)
|
||||
return out
|
||||
|
||||
if not dtype16:
|
||||
# fp32: exact fp32 math, plain Triton elementwise
|
||||
numel = residual.numel()
|
||||
hid = residual.shape[-1]
|
||||
if gate.shape == residual.shape:
|
||||
mode = 0
|
||||
elif _is_row_broadcast_gate(residual, gate):
|
||||
mode = 1
|
||||
else:
|
||||
mode = 2
|
||||
if numel <= 8192:
|
||||
_rga_flat[(triton.cdiv(numel, 256),)](
|
||||
out,
|
||||
residual,
|
||||
update,
|
||||
gate,
|
||||
numel,
|
||||
hid,
|
||||
mode,
|
||||
False,
|
||||
False,
|
||||
BLK=256,
|
||||
num_warps=4,
|
||||
)
|
||||
else:
|
||||
_rga_flat[(triton.cdiv(numel, 1024),)](
|
||||
out,
|
||||
residual,
|
||||
update,
|
||||
gate,
|
||||
numel,
|
||||
hid,
|
||||
mode,
|
||||
False,
|
||||
False,
|
||||
BLK=1024,
|
||||
num_warps=4,
|
||||
)
|
||||
return out
|
||||
|
||||
hid = residual.shape[-1]
|
||||
numel = residual.numel()
|
||||
if _is_row_broadcast_gate(residual, gate):
|
||||
rows = numel // hid
|
||||
if hid >= 4096 or (rows <= 512 and numel >= 1 << 20):
|
||||
_rga_flat_m1[(triton.cdiv(numel, 1024),)](
|
||||
out, residual, update, gate, numel, hid, is_bf16, BLK=1024, num_warps=4
|
||||
)
|
||||
elif hid <= 16384:
|
||||
blk = triton.next_power_of_2(hid)
|
||||
_rga_row[(rows,)](
|
||||
out,
|
||||
residual,
|
||||
update,
|
||||
gate,
|
||||
hid,
|
||||
is_bf16,
|
||||
hid < 4608,
|
||||
hid >= 4096,
|
||||
BLK=blk,
|
||||
num_warps=4 if hid >= 4608 else 8,
|
||||
)
|
||||
else:
|
||||
_rga_flat[(triton.cdiv(numel, 1024),)](
|
||||
out,
|
||||
residual,
|
||||
update,
|
||||
gate,
|
||||
numel,
|
||||
hid,
|
||||
1,
|
||||
is_bf16,
|
||||
True,
|
||||
BLK=1024,
|
||||
num_warps=4,
|
||||
)
|
||||
return out
|
||||
mode = 0 if gate.shape == residual.shape else 2
|
||||
if numel <= 8192:
|
||||
_rga_flat[(triton.cdiv(numel, 256),)](
|
||||
out,
|
||||
residual,
|
||||
update,
|
||||
gate,
|
||||
numel,
|
||||
hid,
|
||||
mode,
|
||||
is_bf16,
|
||||
True,
|
||||
BLK=256,
|
||||
num_warps=4,
|
||||
)
|
||||
batch, tokens, hidden = residual.shape
|
||||
if residual.numel() <= 65536:
|
||||
tile, warps = 16, 4
|
||||
elif residual.numel() >= 1 << 20:
|
||||
tile, warps = 64, 8
|
||||
else:
|
||||
_rga_flat[(triton.cdiv(numel, 1024),)](
|
||||
out,
|
||||
residual,
|
||||
update,
|
||||
gate,
|
||||
numel,
|
||||
hid,
|
||||
mode,
|
||||
is_bf16,
|
||||
True,
|
||||
BLK=1024,
|
||||
num_warps=4,
|
||||
)
|
||||
tile, warps = 32, 8
|
||||
grid = (triton.cdiv(tokens, tile), triton.cdiv(hidden, tile), batch)
|
||||
_rga_transposed[grid](
|
||||
out,
|
||||
residual,
|
||||
update,
|
||||
gate,
|
||||
tokens,
|
||||
hidden,
|
||||
residual.dtype == torch.bfloat16,
|
||||
residual.dtype in (torch.float16, torch.bfloat16),
|
||||
TILE=tile,
|
||||
num_warps=warps,
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
@@ -424,7 +183,7 @@ def _residual_gate_add_custom_op(
|
||||
# benchmarked on. For the contiguous layouts every other diffusion model
|
||||
# uses, the existing JIT CUDA kernel is faster at every measured shape.
|
||||
if _is_transposed_dense_residual(residual, update, gate):
|
||||
return _residual_gate_add_triton(residual, update, gate)
|
||||
return _residual_gate_add_transposed(residual, update, gate)
|
||||
return _residual_gate_add_cuda_impl(residual, update, gate)
|
||||
|
||||
|
||||
@@ -472,7 +231,8 @@ def can_use_residual_gate_add_cuda(
|
||||
residual: torch.Tensor, update: torch.Tensor, gate: torch.Tensor
|
||||
) -> bool:
|
||||
return (
|
||||
residual.dtype in _SUPPORTED_DTYPES
|
||||
torch.version.hip is None
|
||||
and residual.dtype in _SUPPORTED_DTYPES
|
||||
and residual.dtype == update.dtype
|
||||
and residual.dtype == gate.dtype
|
||||
and residual.is_cuda
|
||||
@@ -506,29 +266,11 @@ def residual_gate_add_cuda(
|
||||
def residual_gate_add(
|
||||
residual: torch.Tensor, update: torch.Tensor, gate: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
"""Use the bit-exact CUDA fast path when supported, otherwise eager.
|
||||
|
||||
Runtime build failures are cached per device and dtype so every diffusion
|
||||
model shares one fallback policy instead of maintaining model-local flags.
|
||||
"""
|
||||
runtime_key = (residual.device.index, residual.dtype)
|
||||
if (
|
||||
residual.dtype in _BIT_EXACT_DTYPES
|
||||
and runtime_key not in _FAILED_RUNTIME_KEYS
|
||||
and can_use_residual_gate_add_cuda(residual, update, gate)
|
||||
"""Use the bit-exact fast path for supported layouts, otherwise eager."""
|
||||
if residual.dtype in _BIT_EXACT_DTYPES and can_use_residual_gate_add_cuda(
|
||||
residual, update, gate
|
||||
):
|
||||
try:
|
||||
return _residual_gate_add_custom_op(residual, update, gate)
|
||||
except Exception as exc:
|
||||
if torch.compiler.is_compiling():
|
||||
raise
|
||||
_FAILED_RUNTIME_KEYS.add(runtime_key)
|
||||
logger.warning(
|
||||
"Disabling diffusion residual-gate CUDA fast path on %s/%s: %s",
|
||||
residual.device,
|
||||
residual.dtype,
|
||||
exc,
|
||||
)
|
||||
return _residual_gate_add_custom_op(residual, update, gate)
|
||||
return residual + update * gate
|
||||
|
||||
|
||||
|
||||
@@ -80,18 +80,41 @@ GEMM in FP32. The default path still promotes Q/K/V before both GEMMs.
|
||||
|
||||
## Entry-point protocol
|
||||
|
||||
Every public kernel is a **predicate + kernel** pair:
|
||||
Keep backend selection separate from input validation.
|
||||
|
||||
```python
|
||||
if can_use_<op>(...):
|
||||
out = <op>(...)
|
||||
else:
|
||||
out = <reference chain>
|
||||
```
|
||||
- Select a backend using properties that distinguish supported implementations:
|
||||
device family, dtype, layout, alignment, architecture, and numerical contract.
|
||||
A shape restriction belongs here only when it is a real specialization, such
|
||||
as a fixed head dimension or an unsupported broadcast mode.
|
||||
Use the input's device when selecting CPU versus CUDA: a CUDA-capable
|
||||
process can still receive CPU tensors. A CPU fallback must not disable a
|
||||
process-wide GPU fusion gate.
|
||||
- Validate the selected implementation's inputs before launching. For JIT CUDA,
|
||||
use `TensorMatcher` and `CHECK_HOST` in the C++ launcher; keep dtype checks in
|
||||
the cached module factory. For Triton, validate in the Python launch wrapper.
|
||||
Mismatched shapes or devices are errors, not reasons to silently try another
|
||||
backend. Preserve tensor dimensions until they have been validated.
|
||||
- Add a `can_use_*` predicate only when a caller needs a backend choice. A direct
|
||||
kernel entry point does not need a second copy of the same predicate. Keep
|
||||
fallback selection in one wrapper when several callers share it.
|
||||
- Let errors from a selected implementation propagate. Do not catch every
|
||||
exception and permanently disable a kernel for later requests. First-use
|
||||
numerical verification in `sites/` is a separate policy: retain its reference
|
||||
comparison when exactness depends on a library's reduction or rounding order.
|
||||
|
||||
The kernel raises on an unsupported input. It does not return `None` — a
|
||||
silent `None` is too easy to forget to check, and the failure mode is a
|
||||
wrong-looking image rather than an exception.
|
||||
For example, Helios selects JIT CUDA RoPE on NVIDIA GPUs unless it uses TP
|
||||
RMSNorm. The launcher validates the original batched Q/K and frequencies,
|
||||
including shape relationships, dtype, device, layout and pair alignment.
|
||||
There is no Python `can_use_helios_qk_rope` scan before the call.
|
||||
|
||||
A numerical specialization query should take configuration values, not a list
|
||||
of tensors. For example, `can_use_fused_rmsnorm_scale_shift(dtype, hidden)`
|
||||
selects BF16 with width 2048 or 4096 because those are the FlashInfer reduction
|
||||
trees the kernel reproduces. Its launcher validates weight, scale and shift.
|
||||
|
||||
Existing `try_*` entry points return `None` for an unsupported specialization.
|
||||
Keep that convention explicit at their call sites; do not add it to direct
|
||||
kernel entry points.
|
||||
|
||||
## Selection matrix
|
||||
|
||||
@@ -127,13 +150,12 @@ Several norms look interchangeable and are not. Start here.
|
||||
|
||||
| Entry point | Backend | Contract | Applies to |
|
||||
|---|---|---|---|
|
||||
| `residual_gate_add` | KDA (JIT CUDA) | bit-exact `residual + update * gate` | contiguous tensors, or a transposed-dense `[B, tokens, hidden]` residual/output with contiguous update and row-broadcast gate (SANA-Video) |
|
||||
| `residual_gate_add` | JIT CUDA (contiguous), Triton (transposed) | bit-exact `residual + update * gate` | contiguous tensors, or a transposed-dense `[B, tokens, hidden]` residual/output with contiguous update and row-broadcast gate (SANA-Video) |
|
||||
|
||||
The transposed-dense path uses a shared-memory tile to read the update in
|
||||
logical row-major order while keeping residual reads and output writes
|
||||
coalesced in their `[B, hidden, tokens]` backing layout. Do not insert a
|
||||
`.contiguous()` merely to reach the ordinary path; that restores an entire
|
||||
tensor copy per residual site.
|
||||
The transposed-dense path tiles along the physical stride-1 token dimension
|
||||
for coalesced residual reads and output writes. The contiguous layouts use
|
||||
JIT CUDA. Do not insert a `.contiguous()` to reach that path: it adds a full
|
||||
tensor copy at each residual site.
|
||||
|
||||
### RoPE / QK-norm
|
||||
|
||||
@@ -146,8 +168,8 @@ tensor copy per residual site.
|
||||
| `try_fused_qwen_qkv_epilogue` | JIT CUDA | bit-exact vs the selected BF16 chain | Qwen-Image QK RMSNorm + RoPE + joint QKV writes; SM90+ |
|
||||
| `fused_rope_rotate_half_bitexact` | Triton | bit-exact (elementwise only) |
|
||||
| `fused_complex_rope` | Triton | preserves CUDA complex64 multiply rounding for contiguous BSHD inputs; Qwen-Image 2.1 verifies its first call against eager |
|
||||
| `rmsnorm_preserve_reduction` | Triton + aten | preserves the FP32 mean reduction and cast-before-weight rounding; fuses only pointwise work for contiguous FP16/BF16 inputs; Qwen-Image 2.1 verifies its first call |
|
||||
| `qknorm_complex_rope_cuda` / `qknorm_complex_rope_pack_` | JIT CUDA | bit-exact vs `RMSNorm(cast_x_before_out_mul=True)` + complex64 RoPE for head_dim 128; the pack variant normalizes Q and K in place behind a prefix, copies prefix K/V (and optionally V) in the same launch, and accepts token-strided views of a packed QKV buffer; Qwen-Image 2.1 verifies its first call |
|
||||
| `rmsnorm_preserve_reduction` | Triton + aten | preserves the FP32 mean reduction and cast-before-weight rounding; fuses only pointwise work for FP16/BF16 inputs and materializes strided inputs as contiguous; Qwen-Image 2.1 verifies its first call |
|
||||
| `qknorm_complex_rope_cuda` / `qknorm_complex_rope_pack_` | JIT CUDA | bit-exact vs `RMSNorm(cast_x_before_out_mul=True)` + complex64 RoPE for head_dim 128; both accept token-strided views of a packed QKV buffer; the pack variant normalizes Q and K in place behind a prefix and copies prefix K/V (and optionally V) in the same launch; Qwen-Image 2.1 verifies its first call |
|
||||
| `fused_interleaved_rope_fp64` | JIT CUDA | bit-exact vs paired SANA-Video fp64 RoPE |
|
||||
| `fused_inplace_helios_qk_rope` | JIT CUDA | bit-exact paired in-place RoPE for Helios' transposed frequency layout |
|
||||
| `ltx2_qknorm_split_rope_cuda` | KDA (JIT CUDA) | close; **validated on B200** |
|
||||
@@ -218,7 +240,8 @@ inspecting model modules is its whole job.
|
||||
with its source revision and any JIT CUDA source files.
|
||||
2. Export it from `__init__.py` (`_EXPORTS`) and register a `KernelSpec`
|
||||
(`_SPECS`).
|
||||
3. Give it a `can_use_*` predicate; raise, don't return `None`.
|
||||
3. Keep selection and validation separate as described above. Reuse an existing
|
||||
wrapper when the fallback policy is shared.
|
||||
4. State the numerical contract in the module docstring, including which
|
||||
shapes it was verified on.
|
||||
5. If it is not bit-exact, gate it through `sites/`. It must mount for both
|
||||
|
||||
@@ -564,7 +564,6 @@ for _op, _backend, _target, _caps, _description in _SPECS:
|
||||
_EXPORTS: dict[str, str] = {
|
||||
"can_use_fp8_rowwise": "quantization.fp8_rowwise_triton",
|
||||
"fp8_rowwise": "quantization.fp8_rowwise_triton",
|
||||
"can_use_fused_gelu_tanh_cat": "activation.gelu_tanh_cat_jit",
|
||||
"fused_gelu_tanh_cat": "activation.gelu_tanh_cat_jit",
|
||||
"load_extension_with_recovery": "sglang.srt.utils.cpp_extension_loader",
|
||||
# Normalization: RMSNorm / LayerNorm / GroupNorm and their fused epilogues
|
||||
@@ -598,10 +597,8 @@ _EXPORTS: dict[str, str] = {
|
||||
"triton_one_pass_rms_norm": "norm.rmsnorm_onepass_triton",
|
||||
"can_use_flux2_strided_qknorm_rope": "rope.flux2_qknorm_rope_triton",
|
||||
"flux2_strided_qknorm_rope": "rope.flux2_qknorm_rope_triton",
|
||||
"can_use_rmsnorm_preserve_reduction": "norm.rmsnorm_preserve_reduction",
|
||||
"rmsnorm_preserve_reduction": "norm.rmsnorm_preserve_reduction",
|
||||
"can_use_fused_rmsnorm_scale_shift": "norm.rmsnorm_scale_shift_bitexact",
|
||||
"can_use_fused_scale_residual_rmsnorm_scale_shift": "norm.rmsnorm_scale_shift_bitexact",
|
||||
"fused_rmsnorm_scale_shift_bitexact": "norm.rmsnorm_scale_shift_bitexact",
|
||||
"fused_scale_residual_rmsnorm_scale_shift_bitexact": "norm.rmsnorm_scale_shift_bitexact",
|
||||
"fused_norm_scale_shift": "norm.scale_residual_norm_cutedsl",
|
||||
@@ -636,43 +633,32 @@ _EXPORTS: dict[str, str] = {
|
||||
"try_fused_scaled_residual_bf16": "modulate.scale_shift_triton",
|
||||
"try_fused_scaled_residual_add_exact": "modulate.scale_shift_triton",
|
||||
"timestep_embedding": "modulate.timestep_embedding_jit",
|
||||
"can_use_fused_temb_table_slices": "modulate.wan_temb_table_slices_triton",
|
||||
"fused_temb_table_slices": "modulate.wan_temb_table_slices_triton",
|
||||
# Rotary embeddings and the QK-norm chains fused around them
|
||||
"try_fused_flux2_qkv_epilogue": "sglang.kernels.kda_kernels.flux2_qkv_epilogue_jit",
|
||||
"hunyuan_qkv_rope_pack": "rope.hunyuan_qkv_pack_triton",
|
||||
"can_use_joint_qkv_cat": "layout.joint_qkv_cat_triton",
|
||||
"joint_qkv_cat": "layout.joint_qkv_cat_triton",
|
||||
"can_use_ltx2_qknorm_split_rope_cuda": "sglang.kernels.kda_kernels.ltx2_qknorm_split_rope_jit",
|
||||
"ltx2_qknorm_split_rope_cuda": "sglang.kernels.kda_kernels.ltx2_qknorm_split_rope_jit",
|
||||
"apply_ltx2_split_rotary_emb": "rope.ltx2_rotary_triton",
|
||||
"can_use_ltx25_decoder_rope": "rope.ltx25_decoder_rope_jit",
|
||||
"fused_ltx25_decoder_rope": "rope.ltx25_decoder_rope_jit",
|
||||
"can_use_fused_inplace_qknorm_rope": "rope.qknorm_rope_jit",
|
||||
"fused_inplace_qknorm_rope": "rope.qknorm_rope_jit",
|
||||
"fused_qknorm_rope_pack_kv": "rope.qknorm_rope_jit",
|
||||
"try_fused_qwen_qkv_epilogue": "rope.qwen_qkv_epilogue_jit",
|
||||
"can_use_fused_rope_rotate_half": "rope.rope_rotate_half_bitexact",
|
||||
"fused_rope_rotate_half_bitexact": "rope.rope_rotate_half_bitexact",
|
||||
"can_use_interleaved_rope_fp64": "rope.interleaved_rope_fp64_jit",
|
||||
"fused_interleaved_rope_fp64": "rope.interleaved_rope_fp64_jit",
|
||||
"can_use_helios_qk_rope": "rope.helios_qk_rope_jit",
|
||||
"fused_inplace_helios_qk_rope": "rope.helios_qk_rope_jit",
|
||||
"apply_rotary_embedding": "rope.rotary_triton",
|
||||
"can_use_fused_complex_rope": "rope.complex_rope_triton",
|
||||
"fused_complex_rope": "rope.complex_rope_triton",
|
||||
"can_use_qknorm_complex_rope_cuda": "rope.qknorm_complex_rope_jit",
|
||||
"can_use_qknorm_complex_rope_pack": "rope.qknorm_complex_rope_jit",
|
||||
"qknorm_complex_rope_cuda": "rope.qknorm_complex_rope_jit",
|
||||
"qknorm_complex_rope_pack_": "rope.qknorm_complex_rope_jit",
|
||||
# Tensor layout transformations fused with downstream quantization
|
||||
"try_flux2_token_cat_fp8": "sglang.kernels.kda_kernels.flux2_token_cat_fp8_triton",
|
||||
# Activation-function fusions
|
||||
"can_use_fused_bias_glu": "activation.sana_conv_post_triton",
|
||||
"can_use_fused_bias_silu": "activation.sana_conv_post_triton",
|
||||
"fused_bias_glu": "activation.sana_conv_post_triton",
|
||||
"fused_bias_silu": "activation.sana_conv_post_triton",
|
||||
"can_use_fused_silu_mul": "activation.silu_mul_bitexact",
|
||||
"fused_packed_silu_mul_bitexact": "activation.silu_mul_bitexact",
|
||||
"fused_silu_mul_bitexact": "activation.silu_mul_bitexact",
|
||||
# Diffusion attention kernels
|
||||
@@ -757,7 +743,6 @@ _EXPORTS: dict[str, str] = {
|
||||
"mount_hunyuan_qknorm": "sites.hunyuan_qknorm_site",
|
||||
"try_hunyuan_qknorm": "sites.hunyuan_qknorm_site",
|
||||
"unmount_hunyuan_qknorm": "sites.hunyuan_qknorm_site",
|
||||
"can_use_ltx2_rms_norm_modulate": "sites.ltx2_rmsnorm_modulate_site",
|
||||
"fused_ltx2_rms_norm_modulate": "sites.ltx2_rmsnorm_modulate_site",
|
||||
"ltx2_rms_norm_modulate_active": "sites.ltx2_rmsnorm_modulate_site",
|
||||
"mark_ltx2_rms_norm_modulate_site": "sites.ltx2_rmsnorm_modulate_site",
|
||||
|
||||
@@ -24,34 +24,9 @@ def _module():
|
||||
)
|
||||
|
||||
|
||||
def can_use_fused_gelu_tanh_cat(attn: torch.Tensor, mlp: torch.Tensor) -> bool:
|
||||
return (
|
||||
not torch.is_grad_enabled()
|
||||
and not torch.compiler.is_compiling()
|
||||
and torch.version.hip is None
|
||||
and attn.is_cuda
|
||||
and attn.dtype == mlp.dtype == torch.bfloat16
|
||||
and attn.device == mlp.device
|
||||
and attn.ndim >= 2
|
||||
and attn.shape[:-1] == mlp.shape[:-1]
|
||||
and attn.shape[-1] > 0
|
||||
and mlp.shape[-1] > 0
|
||||
and attn.shape[-1] % 8 == mlp.shape[-1] % 8 == 0
|
||||
and attn.is_contiguous()
|
||||
and mlp.is_contiguous()
|
||||
and attn.numel() > 0
|
||||
and (attn.numel() + mlp.numel()) // 8 <= 2**31 - 1
|
||||
and attn.data_ptr() % 16 == mlp.data_ptr() % 16 == 0
|
||||
)
|
||||
|
||||
|
||||
@register_custom_op(mutates_args=["output"])
|
||||
def _gelu_tanh_cat(attn: torch.Tensor, mlp: torch.Tensor, output: torch.Tensor) -> None:
|
||||
_module().gelu_tanh_cat(
|
||||
attn.view(-1, attn.shape[-1]),
|
||||
mlp.view(-1, mlp.shape[-1]),
|
||||
output.view(-1, output.shape[-1]),
|
||||
)
|
||||
_module().gelu_tanh_cat(attn, mlp, output)
|
||||
|
||||
|
||||
def fused_gelu_tanh_cat(attn: torch.Tensor, mlp: torch.Tensor) -> torch.Tensor:
|
||||
|
||||
@@ -72,31 +72,30 @@ def _bias_glu_kernel(
|
||||
tl.store(out_ptr + offsets, hidden * gate, mask=mask)
|
||||
|
||||
|
||||
def _is_dense_bf16(x: torch.Tensor) -> bool:
|
||||
return (
|
||||
def _validate_conv_post(x: torch.Tensor, bias: torch.Tensor | None) -> None:
|
||||
if not (
|
||||
x.is_cuda
|
||||
and x.dtype is torch.bfloat16
|
||||
and x.dim() == 4
|
||||
and x.ndim == 4
|
||||
and x.numel() > 0
|
||||
and (x.is_contiguous() or x.is_contiguous(memory_format=torch.channels_last))
|
||||
)
|
||||
|
||||
|
||||
def can_use_fused_bias_silu(x: torch.Tensor, bias: torch.Tensor) -> bool:
|
||||
return (
|
||||
_is_dense_bf16(x)
|
||||
and bias.is_cuda
|
||||
and bias.dtype is x.dtype
|
||||
):
|
||||
raise RuntimeError(
|
||||
"Sana conv post-processing expects a dense BF16 CUDA NCHW tensor"
|
||||
)
|
||||
if bias is not None and not (
|
||||
bias.dtype == x.dtype
|
||||
and bias.device == x.device
|
||||
and bias.dim() == 1
|
||||
and bias.shape[0] == x.shape[1]
|
||||
and bias.shape == (x.shape[1],)
|
||||
and bias.is_contiguous()
|
||||
)
|
||||
):
|
||||
raise RuntimeError(
|
||||
"bias must be a contiguous channel vector matching x's dtype and device"
|
||||
)
|
||||
|
||||
|
||||
def fused_bias_silu(x: torch.Tensor, bias: torch.Tensor) -> torch.Tensor:
|
||||
if not can_use_fused_bias_silu(x, bias):
|
||||
raise RuntimeError("unsupported input for Sana fused bias-SiLU")
|
||||
_validate_conv_post(x, bias)
|
||||
out = torch.empty_like(x, memory_format=torch.preserve_format)
|
||||
with torch.cuda.device(x.device):
|
||||
_bias_silu_kernel[(triton.cdiv(x.numel(), 1024),)](
|
||||
@@ -111,32 +110,15 @@ def fused_bias_silu(x: torch.Tensor, bias: torch.Tensor) -> torch.Tensor:
|
||||
return out
|
||||
|
||||
|
||||
def can_use_fused_bias_glu(x: torch.Tensor, bias: torch.Tensor | None) -> bool:
|
||||
return (
|
||||
_is_dense_bf16(x)
|
||||
and x.shape[1] % 2 == 0
|
||||
and (
|
||||
bias is None
|
||||
or (
|
||||
bias.is_cuda
|
||||
and bias.dtype is x.dtype
|
||||
and bias.device == x.device
|
||||
and bias.dim() == 1
|
||||
and bias.shape[0] == x.shape[1]
|
||||
and bias.is_contiguous()
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def fused_bias_glu(x: torch.Tensor, bias: torch.Tensor | None) -> torch.Tensor:
|
||||
"""Apply optional bias then ``hidden * silu(gate)`` along the channel axis.
|
||||
|
||||
Pass no bias for an already biased native depthwise-convolution output:
|
||||
splitting that convolution's bias can change its accumulation rounding.
|
||||
"""
|
||||
if not can_use_fused_bias_glu(x, bias):
|
||||
raise RuntimeError("unsupported input for Sana fused bias-GLU")
|
||||
_validate_conv_post(x, bias)
|
||||
if x.shape[1] % 2:
|
||||
raise RuntimeError("Sana GLU requires an even channel count")
|
||||
batch, double_channels, height, width = x.shape
|
||||
channels = double_channels // 2
|
||||
channels_last = x.is_contiguous(memory_format=torch.channels_last)
|
||||
@@ -161,8 +143,6 @@ def fused_bias_glu(x: torch.Tensor, bias: torch.Tensor | None) -> torch.Tensor:
|
||||
|
||||
|
||||
__all__ = [
|
||||
"can_use_fused_bias_glu",
|
||||
"can_use_fused_bias_silu",
|
||||
"fused_bias_glu",
|
||||
"fused_bias_silu",
|
||||
]
|
||||
|
||||
@@ -60,20 +60,6 @@ def _packed_silu_mul_kernel(
|
||||
tl.store(out_ptr + row * D + cols, s * b, mask=mask)
|
||||
|
||||
|
||||
def can_use_fused_silu_mul(a: torch.Tensor, b: torch.Tensor) -> bool:
|
||||
return (
|
||||
a.dtype is torch.bfloat16
|
||||
and b.dtype is torch.bfloat16
|
||||
and a.is_cuda
|
||||
and b.is_cuda
|
||||
and a.device == b.device
|
||||
and a.shape == b.shape
|
||||
and a.is_contiguous()
|
||||
and b.is_contiguous()
|
||||
and a.numel() > 0
|
||||
)
|
||||
|
||||
|
||||
def _fake_silu_mul(a: torch.Tensor, b: torch.Tensor) -> torch.Tensor:
|
||||
return torch.empty_like(a)
|
||||
|
||||
@@ -85,6 +71,18 @@ def _fake_silu_mul(a: torch.Tensor, b: torch.Tensor) -> torch.Tensor:
|
||||
)
|
||||
def fused_silu_mul_bitexact(a: torch.Tensor, b: torch.Tensor) -> torch.Tensor:
|
||||
"""``silu(a) * b``, bit-exact vs the eager two-kernel chain."""
|
||||
if not (
|
||||
a.dtype is torch.bfloat16
|
||||
and b.dtype is torch.bfloat16
|
||||
and a.is_cuda
|
||||
and b.is_cuda
|
||||
and a.device == b.device
|
||||
and a.shape == b.shape
|
||||
and a.is_contiguous()
|
||||
and b.is_contiguous()
|
||||
and a.numel() > 0
|
||||
):
|
||||
raise RuntimeError("invalid input for fused_silu_mul_bitexact")
|
||||
out = torch.empty_like(a)
|
||||
numel = a.numel()
|
||||
with torch.cuda.device(a.device):
|
||||
|
||||
@@ -65,34 +65,22 @@ def vdn_delta_factors(
|
||||
transition = torch.empty_like(A)
|
||||
injection = torch.empty_like(B)
|
||||
module = _jit_vdn_delta_factors_module()
|
||||
module.vdn_delta_factors(
|
||||
transition.view(-1, HEAD_DIM, HEAD_DIM),
|
||||
injection.view(-1, HEAD_DIM, HEAD_DIM),
|
||||
A.view(-1, HEAD_DIM, HEAD_DIM),
|
||||
B.view(-1, HEAD_DIM, HEAD_DIM),
|
||||
alpha.view(-1, HEAD_DIM),
|
||||
)
|
||||
module.vdn_delta_factors(transition, injection, A, B, alpha)
|
||||
return transition, injection
|
||||
|
||||
|
||||
@torch.compiler.assume_constant_result
|
||||
@cache_once
|
||||
def can_use_vdn_delta_factors(
|
||||
A: torch.Tensor, B: torch.Tensor, alpha: torch.Tensor
|
||||
device: torch.device, dtype: torch.dtype, head_dim: int
|
||||
) -> bool:
|
||||
"""The register inverse is specialized to FP32, 128-wide heads on SM80+."""
|
||||
return (
|
||||
A.is_cuda
|
||||
and A.dtype is torch.float32
|
||||
and B.dtype is torch.float32
|
||||
and alpha.dtype is torch.float32
|
||||
and A.device == B.device == alpha.device
|
||||
and A.dim() >= 2
|
||||
and A.shape[-1] == HEAD_DIM
|
||||
and A.shape[-2] == HEAD_DIM
|
||||
and B.shape == A.shape
|
||||
and alpha.shape == A.shape[:-1]
|
||||
and A.is_contiguous()
|
||||
and B.is_contiguous()
|
||||
and alpha.is_contiguous()
|
||||
and torch.cuda.get_device_capability(A.device)[0] >= 8
|
||||
device.type == "cuda"
|
||||
and not torch.version.hip
|
||||
and dtype is torch.float32
|
||||
and head_dim == HEAD_DIM
|
||||
and torch.cuda.get_device_capability(device)[0] >= 8
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -55,35 +55,6 @@ def _joint_qkv_cat_kernel(
|
||||
tl.store(OUT + (component * BATCH * sequence + row) * HIDDEN + cols, value, mask)
|
||||
|
||||
|
||||
def can_use_joint_qkv_cat(*inputs: torch.Tensor) -> bool:
|
||||
if len(inputs) != 6 or torch.compiler.is_compiling() or torch.version.hip:
|
||||
return False
|
||||
first = inputs[0]
|
||||
if first.ndim != 4 or not first.is_cuda:
|
||||
return False
|
||||
if first.dtype not in (torch.float16, torch.bfloat16):
|
||||
return False
|
||||
batch, tokens, heads, dim = first.shape
|
||||
if min(batch, tokens, heads, dim) <= 0 or heads * dim > 8192:
|
||||
return False
|
||||
for index, value in enumerate(inputs):
|
||||
expected = first.shape if index < 3 else inputs[3].shape
|
||||
if (
|
||||
value.ndim != 4
|
||||
or value.shape != expected
|
||||
or value.shape[0] != batch
|
||||
or value.shape[2:] != (heads, dim)
|
||||
or value.shape[1] <= 0
|
||||
or value.dtype != first.dtype
|
||||
or value.device != first.device
|
||||
or value.requires_grad
|
||||
or value.stride(-1) != 1
|
||||
or value.stride(-2) != dim
|
||||
):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def joint_qkv_cat(
|
||||
img_q: torch.Tensor,
|
||||
img_k: torch.Tensor,
|
||||
@@ -98,7 +69,33 @@ def joint_qkv_cat(
|
||||
The three outputs occupy disjoint, contiguous regions of one allocation.
|
||||
"""
|
||||
inputs = (img_q, img_k, img_v, txt_q, txt_k, txt_v)
|
||||
assert can_use_joint_qkv_cat(*inputs)
|
||||
if not (
|
||||
img_q.is_cuda
|
||||
and img_q.dtype in (torch.float16, torch.bfloat16)
|
||||
and img_q.ndim == 4
|
||||
and min(img_q.shape) > 0
|
||||
and img_q.shape[-2] * img_q.shape[-1] <= 8192
|
||||
):
|
||||
raise RuntimeError(
|
||||
"joint QKV expects FP16/BF16 CUDA [B, S, H, D], 0 < H * D <= 8192"
|
||||
)
|
||||
for index, value in enumerate(inputs):
|
||||
expected = img_q.shape if index < 3 else txt_q.shape
|
||||
if not (
|
||||
value.ndim == 4
|
||||
and value.shape == expected
|
||||
and value.shape[0] == img_q.shape[0]
|
||||
and value.shape[2:] == img_q.shape[2:]
|
||||
and value.shape[1] > 0
|
||||
and value.dtype == img_q.dtype
|
||||
and value.device == img_q.device
|
||||
and not value.requires_grad
|
||||
and value.stride(-1) == 1
|
||||
and value.stride(-2) == img_q.shape[-1]
|
||||
):
|
||||
raise RuntimeError(
|
||||
"joint QKV inputs must share batch/head shape, dtype/device and have packed head rows without gradients"
|
||||
)
|
||||
batch, image_tokens, heads, dim = img_q.shape
|
||||
text_tokens = txt_q.shape[1]
|
||||
output = torch.empty(
|
||||
|
||||
@@ -42,7 +42,7 @@ def _fake_merge_heads(x: torch.Tensor) -> torch.Tensor:
|
||||
mutates_args=[],
|
||||
fake_impl=_fake_merge_heads,
|
||||
)
|
||||
def _usp_merge_heads_custom_op(x: torch.Tensor) -> torch.Tensor:
|
||||
def _usp_merge_heads_cuda(x: torch.Tensor) -> torch.Tensor:
|
||||
world, seq, batch, h_local, head_dim = x.shape
|
||||
out = x.new_empty((batch, seq, world, h_local, head_dim))
|
||||
module = _jit_usp_relayout_module(x.dtype)
|
||||
@@ -52,27 +52,14 @@ def _usp_merge_heads_custom_op(x: torch.Tensor) -> torch.Tensor:
|
||||
|
||||
def can_use_usp_merge_heads(x: torch.Tensor) -> bool:
|
||||
return (
|
||||
isinstance(x, torch.Tensor)
|
||||
and torch.version.hip is None
|
||||
torch.version.hip is None
|
||||
and x.is_cuda
|
||||
and x.dtype in _SUPPORTED_DTYPES
|
||||
and x.dim() == 5
|
||||
and x.numel() > 0
|
||||
and x.is_contiguous()
|
||||
)
|
||||
|
||||
|
||||
def _usp_merge_heads_cuda(x: torch.Tensor) -> torch.Tensor:
|
||||
"""[W, S, B, h_local, D] -> [B, S, W, h_local, D] contiguous.
|
||||
|
||||
Bit-exact single-pass replacement for
|
||||
``x.permute(2, 1, 0, 3, 4).contiguous()`` on the Ulysses output path.
|
||||
"""
|
||||
if not can_use_usp_merge_heads(x):
|
||||
raise RuntimeError("unsupported input for usp_merge_heads CUDA")
|
||||
return _usp_merge_heads_custom_op(x)
|
||||
|
||||
|
||||
def usp_merge_heads(x: torch.Tensor) -> torch.Tensor:
|
||||
"""Merge Ulysses output heads with an exact eager fallback.
|
||||
|
||||
|
||||
@@ -7,7 +7,6 @@ bit-exact vs the eager chain (``torch.equal``) and needs no quality gate.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
@@ -21,9 +20,6 @@ if TYPE_CHECKING:
|
||||
|
||||
_SUPPORTED_DTYPES = (torch.float16, torch.bfloat16)
|
||||
_ALIGN_BYTES = 16
|
||||
_FAILED_RUNTIME_KEYS: set[tuple[int | None, torch.dtype]] = set()
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@cache_once
|
||||
@@ -55,9 +51,10 @@ def _fake_impl(
|
||||
mutates_args=[],
|
||||
fake_impl=_fake_impl,
|
||||
)
|
||||
def _modulate_scale_shift_custom_op(
|
||||
def modulate_scale_shift_cuda(
|
||||
x: torch.Tensor, scale: torch.Tensor, shift: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
"""Fused ``x * (1 + scale[:, None]) + shift[:, None]``; validated by CUDA."""
|
||||
out = torch.empty_like(x)
|
||||
module = _jit_modulate_scale_shift_module(x.dtype)
|
||||
module.modulate_scale_shift(out, x, scale, shift)
|
||||
@@ -72,7 +69,8 @@ def can_use_modulate_scale_shift_cuda(
|
||||
x: torch.Tensor, scale: torch.Tensor, shift: torch.Tensor
|
||||
) -> bool:
|
||||
if (
|
||||
x.dtype not in _SUPPORTED_DTYPES
|
||||
torch.version.hip is not None
|
||||
or x.dtype not in _SUPPORTED_DTYPES
|
||||
or scale.dtype != x.dtype
|
||||
or shift.dtype != x.dtype
|
||||
or not (x.is_cuda and scale.is_cuda and shift.is_cuda)
|
||||
@@ -91,35 +89,12 @@ def can_use_modulate_scale_shift_cuda(
|
||||
)
|
||||
|
||||
|
||||
def modulate_scale_shift_cuda(
|
||||
x: torch.Tensor, scale: torch.Tensor, shift: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
"""Fused ``x * (1 + scale[:, None]) + shift[:, None]`` (bit-exact vs eager)."""
|
||||
if not can_use_modulate_scale_shift_cuda(x, scale, shift):
|
||||
raise RuntimeError("unsupported input for modulate_scale_shift CUDA")
|
||||
return _modulate_scale_shift_custom_op(x, scale, shift)
|
||||
|
||||
|
||||
def modulate_scale_shift(
|
||||
x: torch.Tensor, scale: torch.Tensor, shift: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
"""Use the bit-exact CUDA fast path when supported, otherwise eager."""
|
||||
runtime_key = (x.device.index, x.dtype)
|
||||
if runtime_key not in _FAILED_RUNTIME_KEYS and can_use_modulate_scale_shift_cuda(
|
||||
x, scale, shift
|
||||
):
|
||||
try:
|
||||
return modulate_scale_shift_cuda(x, scale, shift)
|
||||
except Exception as exc:
|
||||
if torch.compiler.is_compiling():
|
||||
raise
|
||||
_FAILED_RUNTIME_KEYS.add(runtime_key)
|
||||
logger.warning(
|
||||
"Disabling diffusion modulate CUDA fast path on %s/%s: %s",
|
||||
x.device,
|
||||
x.dtype,
|
||||
exc,
|
||||
)
|
||||
if can_use_modulate_scale_shift_cuda(x, scale, shift):
|
||||
return modulate_scale_shift_cuda(x, scale, shift)
|
||||
return x * (1 + scale[:, None]) + shift[:, None]
|
||||
|
||||
|
||||
|
||||
@@ -49,22 +49,6 @@ def _temb_table_slices_kernel(
|
||||
tl.store(out_ptr + (j * rows + row) * D + cols, w + t, mask=mask)
|
||||
|
||||
|
||||
def can_use_fused_temb_table_slices(table: torch.Tensor, temb: torch.Tensor) -> bool:
|
||||
return (
|
||||
temb.is_cuda
|
||||
and temb.dtype in (torch.bfloat16, torch.float16, torch.float32)
|
||||
and temb.dim() == 4
|
||||
and temb.shape[2] == 6
|
||||
and temb.is_contiguous()
|
||||
and table.is_cuda
|
||||
and table.device == temb.device
|
||||
and table.dtype in (torch.bfloat16, torch.float16, torch.float32)
|
||||
and table.shape == (1, 6, temb.shape[-1])
|
||||
and table.is_contiguous()
|
||||
and temb.numel() > 0
|
||||
)
|
||||
|
||||
|
||||
def _fake_temb_table_slices(table: torch.Tensor, temb: torch.Tensor) -> torch.Tensor:
|
||||
batch, seq_len, six, hidden = temb.shape
|
||||
return temb.new_empty((six, batch, seq_len, hidden), dtype=torch.float32)
|
||||
@@ -81,6 +65,20 @@ def fused_temb_table_slices(table: torch.Tensor, temb: torch.Tensor) -> torch.Te
|
||||
``temb`` is ``(B, S, 6, D)``; ``table`` is the block's ``(1, 6, D)`` fp32
|
||||
adaLN table. ``out[j]`` is the ``j``-th modulation slice, contiguous.
|
||||
"""
|
||||
if not (
|
||||
temb.is_cuda
|
||||
and temb.dtype in (torch.bfloat16, torch.float16, torch.float32)
|
||||
and temb.dim() == 4
|
||||
and temb.shape[2] == 6
|
||||
and temb.is_contiguous()
|
||||
and table.is_cuda
|
||||
and table.device == temb.device
|
||||
and table.dtype in (torch.bfloat16, torch.float16, torch.float32)
|
||||
and table.shape == (1, 6, temb.shape[-1])
|
||||
and table.is_contiguous()
|
||||
and temb.numel() > 0
|
||||
):
|
||||
raise RuntimeError("invalid input for fused_temb_table_slices")
|
||||
batch, seq_len, _, hidden = temb.shape
|
||||
out = temb.new_empty((6, batch, seq_len, hidden), dtype=torch.float32)
|
||||
rows = batch * seq_len
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
"""Fuse RMSNorm pointwise work while retaining the native FP32 mean reduction.
|
||||
|
||||
Matches ``weight * (x.float() * rsqrt(mean(x.float()**2) + eps)).to(x.dtype)``
|
||||
for contiguous FP16/BF16 inputs. The FP32 square buffer has the original shape,
|
||||
for FP16/BF16 inputs; packed projection views are materialized before the launch. The FP32 square buffer has the original shape,
|
||||
so aten selects the same reduction as the eager chain. Verified at head width
|
||||
128, including 131072 rows; callers verify their first dispatch before reuse.
|
||||
"""
|
||||
@@ -42,23 +42,8 @@ def _rmsnorm_finish_kernel(
|
||||
tl.store(out_ptr + index, normalized.to(tl.float32) * weight, mask)
|
||||
|
||||
|
||||
def can_use_rmsnorm_preserve_reduction(x: torch.Tensor, weight: torch.Tensor) -> bool:
|
||||
return (
|
||||
x.is_cuda
|
||||
and torch.version.hip is None
|
||||
and x.dtype in (torch.float16, torch.bfloat16)
|
||||
and x.ndim >= 2
|
||||
and x.numel() > 0
|
||||
and x.is_contiguous()
|
||||
and weight.device == x.device
|
||||
and weight.dtype == x.dtype
|
||||
and weight.shape == (x.shape[-1],)
|
||||
and weight.is_contiguous()
|
||||
)
|
||||
|
||||
|
||||
def _fake_rmsnorm(x: torch.Tensor, weight: torch.Tensor, eps: float) -> torch.Tensor:
|
||||
return torch.empty_like(x)
|
||||
return torch.empty(x.shape, dtype=x.dtype, device=x.device)
|
||||
|
||||
|
||||
@register_custom_op(
|
||||
@@ -70,7 +55,19 @@ def rmsnorm_preserve_reduction(
|
||||
x: torch.Tensor, weight: torch.Tensor, eps: float
|
||||
) -> torch.Tensor:
|
||||
"""Preserve aten's mean and cast-before-weight semantics without residuals."""
|
||||
assert can_use_rmsnorm_preserve_reduction(x, weight)
|
||||
if not (
|
||||
x.is_cuda
|
||||
and torch.version.hip is None
|
||||
and x.dtype in (torch.float16, torch.bfloat16)
|
||||
and x.ndim >= 2
|
||||
and x.numel() > 0
|
||||
and weight.device == x.device
|
||||
and weight.dtype == x.dtype
|
||||
and weight.shape == (x.shape[-1],)
|
||||
and weight.is_contiguous()
|
||||
):
|
||||
raise RuntimeError("invalid input for rmsnorm_preserve_reduction")
|
||||
x = x.contiguous()
|
||||
squares = torch.empty_like(x, dtype=torch.float32)
|
||||
out = torch.empty_like(x)
|
||||
with torch.cuda.device(x.device):
|
||||
|
||||
@@ -155,59 +155,57 @@ def _threads_per_row(hidden: int) -> int | None:
|
||||
return tpr
|
||||
|
||||
|
||||
def _is_row_broadcast(t: torch.Tensor, x: torch.Tensor) -> bool:
|
||||
def _is_row_broadcast(t: torch.Tensor, shape: tuple, device: torch.device) -> bool:
|
||||
return (
|
||||
t.dtype is torch.bfloat16
|
||||
and t.is_cuda
|
||||
and t.device == x.device
|
||||
and t.shape == (x.shape[0], 1, x.shape[-1])
|
||||
and t.device == device
|
||||
and t.shape == shape
|
||||
and t.is_contiguous()
|
||||
)
|
||||
|
||||
|
||||
def can_use_fused_rmsnorm_scale_shift(
|
||||
x: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
scale: torch.Tensor,
|
||||
shift: torch.Tensor,
|
||||
) -> bool:
|
||||
def can_use_fused_rmsnorm_scale_shift(dtype: torch.dtype, hidden: int) -> bool:
|
||||
"""Select the BF16 FlashInfer reduction replicated by this kernel."""
|
||||
return (
|
||||
# ROCm cannot compile the inline PTX above: LLVM makes the unusable
|
||||
# `=f` constraint a fatal error that kills the process, so reject
|
||||
# before the first launch rather than rely on the caller's fallback.
|
||||
is_cuda() and dtype is torch.bfloat16 and _threads_per_row(hidden) is not None
|
||||
)
|
||||
|
||||
|
||||
def _validate_rmsnorm_modulate(x, weight, scale, shift) -> tuple[int, int, int, int]:
|
||||
shape = x.shape
|
||||
if not (
|
||||
is_cuda()
|
||||
and x.dtype is torch.bfloat16
|
||||
and x.is_cuda
|
||||
and x.dim() == 3
|
||||
and x.dtype is torch.bfloat16
|
||||
and len(shape) == 3
|
||||
and x.is_contiguous()
|
||||
and _threads_per_row(x.shape[-1]) is not None
|
||||
and weight.dtype is torch.bfloat16
|
||||
and weight.is_cuda
|
||||
and weight.device == x.device
|
||||
and weight.shape == (x.shape[-1],)
|
||||
):
|
||||
raise RuntimeError(
|
||||
"RMSNorm modulation expects contiguous BF16 CUDA [B, S, D], D in (2048, 4096)"
|
||||
)
|
||||
batch, seq_len, hidden = shape
|
||||
tpr = _threads_per_row(hidden)
|
||||
if tpr is None:
|
||||
raise RuntimeError("RMSNorm modulation requires D in (2048, 4096)")
|
||||
device = x.device
|
||||
if not (
|
||||
weight.dtype is torch.bfloat16
|
||||
and weight.device == device
|
||||
and weight.shape == (hidden,)
|
||||
and weight.is_contiguous()
|
||||
and _is_row_broadcast(scale, x)
|
||||
and _is_row_broadcast(shift, x)
|
||||
)
|
||||
|
||||
|
||||
def can_use_fused_scale_residual_rmsnorm_scale_shift(
|
||||
residual: torch.Tensor,
|
||||
update: torch.Tensor,
|
||||
gate: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
scale: torch.Tensor,
|
||||
shift: torch.Tensor,
|
||||
) -> bool:
|
||||
return (
|
||||
can_use_fused_rmsnorm_scale_shift(residual, weight, scale, shift)
|
||||
and update.dtype is torch.bfloat16
|
||||
and update.is_cuda
|
||||
and update.device == residual.device
|
||||
and update.shape == residual.shape
|
||||
and update.is_contiguous()
|
||||
and _is_row_broadcast(gate, residual)
|
||||
)
|
||||
):
|
||||
raise RuntimeError(
|
||||
"weight must be a contiguous [D] vector matching x's dtype/device"
|
||||
)
|
||||
row_shape = (batch, 1, hidden)
|
||||
if not (
|
||||
_is_row_broadcast(scale, row_shape, device)
|
||||
and _is_row_broadcast(shift, row_shape, device)
|
||||
):
|
||||
raise RuntimeError(
|
||||
"scale and shift must be contiguous [B, 1, D] tensors matching x's dtype/device"
|
||||
)
|
||||
return batch, seq_len, hidden, tpr
|
||||
|
||||
|
||||
def _fake_norm_scale_shift(
|
||||
@@ -233,8 +231,7 @@ def fused_rmsnorm_scale_shift_bitexact(
|
||||
eps: float,
|
||||
) -> torch.Tensor:
|
||||
"""``norm(x) * (1 + scale) + shift``, bit-exact vs the eager chain."""
|
||||
batch, seq_len, hidden = x.shape
|
||||
tpr = _threads_per_row(hidden)
|
||||
batch, seq_len, hidden, tpr = _validate_rmsnorm_modulate(x, weight, scale, shift)
|
||||
out = torch.empty_like(x)
|
||||
with torch.cuda.device(x.device):
|
||||
_rmsnorm_scale_shift_kernel[(batch * seq_len,)](
|
||||
@@ -291,8 +288,19 @@ def fused_scale_residual_rmsnorm_scale_shift_bitexact(
|
||||
Returns ``(modulated, res)``, both bit-exact vs the eager pair + norm
|
||||
chain (and therefore vs the ``residual_gate_add_cuda`` fast path).
|
||||
"""
|
||||
batch, seq_len, hidden = residual.shape
|
||||
tpr = _threads_per_row(hidden)
|
||||
batch, seq_len, hidden, tpr = _validate_rmsnorm_modulate(
|
||||
residual, weight, scale, shift
|
||||
)
|
||||
if not (
|
||||
update.dtype == residual.dtype
|
||||
and update.device == residual.device
|
||||
and update.shape == residual.shape
|
||||
and update.is_contiguous()
|
||||
and _is_row_broadcast(gate, (batch, 1, hidden), residual.device)
|
||||
):
|
||||
raise RuntimeError(
|
||||
"update must match residual; gate must be contiguous [B, 1, D] with the same dtype/device"
|
||||
)
|
||||
out = torch.empty_like(residual)
|
||||
res_out = torch.empty_like(residual)
|
||||
with torch.cuda.device(residual.device):
|
||||
|
||||
@@ -50,22 +50,6 @@ def _fuse_real_sin(device: torch.device) -> bool:
|
||||
return (z[0] * z[1]).imag.item() != 0
|
||||
|
||||
|
||||
def can_use_fused_complex_rope(x: torch.Tensor, rope: torch.Tensor) -> bool:
|
||||
return (
|
||||
x.is_cuda
|
||||
and torch.version.hip is None
|
||||
and x.dtype in (torch.float16, torch.bfloat16, torch.float32)
|
||||
and x.ndim == 4
|
||||
and x.numel() > 0
|
||||
and x.shape[-1] % 2 == 0
|
||||
and x.is_contiguous()
|
||||
and rope.dtype == torch.complex64
|
||||
and rope.device == x.device
|
||||
and rope.shape == (x.shape[1], x.shape[-1] // 2)
|
||||
and rope.is_contiguous()
|
||||
)
|
||||
|
||||
|
||||
def _fake_complex_rope(x: torch.Tensor, rope: torch.Tensor) -> torch.Tensor:
|
||||
return torch.empty_like(x)
|
||||
|
||||
@@ -77,7 +61,20 @@ def _fake_complex_rope(x: torch.Tensor, rope: torch.Tensor) -> torch.Tensor:
|
||||
)
|
||||
def fused_complex_rope(x: torch.Tensor, rope: torch.Tensor) -> torch.Tensor:
|
||||
"""Rotate contiguous BSHD activations with a shared S×(D/2) complex cache."""
|
||||
assert can_use_fused_complex_rope(x, rope)
|
||||
if not (
|
||||
x.is_cuda
|
||||
and torch.version.hip is None
|
||||
and x.dtype in (torch.float16, torch.bfloat16, torch.float32)
|
||||
and x.ndim == 4
|
||||
and x.numel() > 0
|
||||
and x.shape[-1] % 2 == 0
|
||||
and x.is_contiguous()
|
||||
and rope.dtype == torch.complex64
|
||||
and rope.device == x.device
|
||||
and rope.shape == (x.shape[1], x.shape[-1] // 2)
|
||||
and rope.is_contiguous()
|
||||
):
|
||||
raise RuntimeError("invalid input for fused_complex_rope")
|
||||
out = torch.empty_like(x)
|
||||
pairs = x.numel() // 2
|
||||
with torch.cuda.device(x.device):
|
||||
|
||||
@@ -1,11 +1,4 @@
|
||||
"""Bit-exact paired RoPE for Helios' transposed frequency layout.
|
||||
|
||||
The JIT kernel preserves the eager path's separate fp32 multiply and add/sub
|
||||
rounding boundaries, then rounds each adjacent pair back to fp16/bf16 in
|
||||
place. It is verified on head dimensions 64, 128, and 256, including Helios'
|
||||
production ``[8640, 40, 128]`` Q/K shape. Unsupported layouts retain the eager
|
||||
model path through :func:`can_use_helios_qk_rope`.
|
||||
"""
|
||||
"""Helios paired RoPE, preserving separate FP32 multiply/add rounding."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -22,10 +15,8 @@ if TYPE_CHECKING:
|
||||
|
||||
@cache_once
|
||||
def _jit_helios_qk_rope_module(dtype: torch.dtype) -> Module:
|
||||
if dtype not in (torch.float16, torch.bfloat16):
|
||||
raise RuntimeError(
|
||||
f"Unsupported Helios QK RoPE dtype {dtype}; expected float16 or bfloat16"
|
||||
)
|
||||
if dtype not in (torch.float16, torch.bfloat16, torch.float32):
|
||||
raise RuntimeError(f"Unsupported Helios QK RoPE dtype: {dtype}")
|
||||
args = make_cpp_args(dtype)
|
||||
return load_jit(
|
||||
"helios_qk_rope",
|
||||
@@ -41,41 +32,13 @@ def fused_inplace_helios_qk_rope(
|
||||
k: torch.Tensor,
|
||||
freqs: torch.Tensor,
|
||||
) -> None:
|
||||
"""Apply Helios' transposed RoPE to contiguous normalized Q/K in place."""
|
||||
"""Rotate contiguous Q/K shaped [N, H, D] or [B, S, H, D] in place.
|
||||
|
||||
Frequencies have shape [N, 2D] or [B, S, 2D]. The C++ launcher
|
||||
validates these shapes, dtypes, devices and pair alignment.
|
||||
"""
|
||||
module = _jit_helios_qk_rope_module(q.dtype)
|
||||
module.helios_qk_rope(q, k, freqs)
|
||||
|
||||
|
||||
def can_use_helios_qk_rope(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
freqs: torch.Tensor,
|
||||
) -> bool:
|
||||
"""Return whether tensors match the native Helios paired-RoPE contract."""
|
||||
if q.dim() != 4 or freqs.dim() != 3:
|
||||
return False
|
||||
# Dynamo cannot trace pointer or storage-offset queries. Compiled Helios Q/K
|
||||
# come directly from aligned linear outputs; eager callers retain the guard.
|
||||
pair_aligned = True
|
||||
if not torch.compiler.is_compiling():
|
||||
pair_aligned = q.storage_offset() % 2 == 0 and k.storage_offset() % 2 == 0
|
||||
return (
|
||||
q.is_cuda
|
||||
and k.is_cuda
|
||||
and freqs.is_cuda
|
||||
and q.dtype in (torch.float16, torch.bfloat16)
|
||||
and k.dtype == q.dtype
|
||||
and freqs.dtype is torch.float32
|
||||
and q.device == k.device == freqs.device
|
||||
and k.shape == q.shape
|
||||
and all(size > 0 for size in q.shape)
|
||||
and freqs.shape == (*q.shape[:2], 2 * q.shape[-1])
|
||||
and q.shape[-1] % 2 == 0
|
||||
and q.is_contiguous()
|
||||
and k.is_contiguous()
|
||||
and freqs.is_contiguous()
|
||||
and pair_aligned
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["can_use_helios_qk_rope", "fused_inplace_helios_qk_rope"]
|
||||
__all__ = ["fused_inplace_helios_qk_rope"]
|
||||
|
||||
@@ -54,52 +54,10 @@ def fused_interleaved_rope_fp64(
|
||||
q_out = torch.empty_like(q)
|
||||
k_out = torch.empty_like(k)
|
||||
module = _jit_interleaved_rope_fp64_module(q.dtype)
|
||||
module.interleaved_rope_fp64(
|
||||
q_out.view(-1),
|
||||
k_out.view(-1),
|
||||
q.view(-1),
|
||||
k.view(-1),
|
||||
cos.view(-1),
|
||||
sin.view(-1),
|
||||
q.shape[0],
|
||||
q.shape[1],
|
||||
q.shape[2],
|
||||
q.shape[3],
|
||||
)
|
||||
module.interleaved_rope_fp64(q_out, k_out, q, k, cos, sin)
|
||||
return q_out, k_out
|
||||
|
||||
|
||||
def can_use_interleaved_rope_fp64(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
cos: torch.Tensor,
|
||||
sin: torch.Tensor,
|
||||
) -> bool:
|
||||
if q.dim() != 4:
|
||||
return False
|
||||
expected_table_shape = (1, q.shape[1], 1, q.shape[3])
|
||||
return (
|
||||
q.dtype is torch.bfloat16
|
||||
and k.dtype is q.dtype
|
||||
and q.is_cuda
|
||||
and k.is_cuda
|
||||
and q.device == k.device == cos.device == sin.device
|
||||
and k.shape == q.shape
|
||||
and q.shape[-1] % 2 == 0
|
||||
and q.is_contiguous()
|
||||
and k.is_contiguous()
|
||||
and q.data_ptr() % 4 == 0
|
||||
and k.data_ptr() % 4 == 0
|
||||
and cos.dtype is torch.float64
|
||||
and sin.dtype is torch.float64
|
||||
and cos.shape == expected_table_shape
|
||||
and sin.shape == expected_table_shape
|
||||
and cos.is_contiguous()
|
||||
and sin.is_contiguous()
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"can_use_interleaved_rope_fp64",
|
||||
"fused_interleaved_rope_fp64",
|
||||
]
|
||||
|
||||
@@ -67,74 +67,11 @@ def fused_ltx25_decoder_rope(
|
||||
k_out = torch.empty_like(k)
|
||||
module = _jit_ltx25_decoder_rope_module(q.dtype)
|
||||
module.ltx25_decoder_rope(
|
||||
q_out.view(-1),
|
||||
k_out.view(-1),
|
||||
q.view(-1),
|
||||
k.view(-1),
|
||||
cos_t.view(-1),
|
||||
sin_t.view(-1),
|
||||
cos_h.view(-1),
|
||||
sin_h.view(-1),
|
||||
cos_w.view(-1),
|
||||
sin_w.view(-1),
|
||||
q.shape[0],
|
||||
q.shape[1],
|
||||
q.shape[2],
|
||||
q.shape[3],
|
||||
q.shape[4],
|
||||
q.shape[5],
|
||||
dim_t,
|
||||
dim_h,
|
||||
q_out, k_out, q, k, cos_t, sin_t, cos_h, sin_h, cos_w, sin_w, dim_t, dim_h
|
||||
)
|
||||
return q_out, k_out
|
||||
|
||||
|
||||
def can_use_ltx25_decoder_rope(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
tables: tuple[tuple[torch.Tensor, torch.Tensor], ...],
|
||||
dim_split: tuple[int, int, int],
|
||||
) -> bool:
|
||||
if (
|
||||
q.dim() != 6
|
||||
or len(tables) != 3
|
||||
or any(len(pair) != 2 for pair in tables)
|
||||
or len(dim_split) != 3
|
||||
):
|
||||
return False
|
||||
dim_t, dim_h, dim_w = dim_split
|
||||
expected_shapes = (
|
||||
(q.shape[1], dim_t // 2),
|
||||
(q.shape[2], dim_h // 2),
|
||||
(q.shape[3], dim_w // 2),
|
||||
)
|
||||
flat_tables = tuple(table for pair in tables for table in pair)
|
||||
return (
|
||||
q.dtype is torch.bfloat16
|
||||
and k.dtype is q.dtype
|
||||
and q.is_cuda
|
||||
and k.is_cuda
|
||||
and q.device == k.device
|
||||
and k.shape == q.shape
|
||||
and all(size > 0 for size in q.shape)
|
||||
and q.shape[-1] == sum(dim_split)
|
||||
and all(dim > 0 and dim % 2 == 0 for dim in dim_split)
|
||||
and q.is_contiguous()
|
||||
and k.is_contiguous()
|
||||
and q.data_ptr() % 4 == 0
|
||||
and k.data_ptr() % 4 == 0
|
||||
and all(table.device == q.device for table in flat_tables)
|
||||
and all(table.dtype is torch.float32 for table in flat_tables)
|
||||
and all(
|
||||
cos.shape == sin.shape == expected_shape
|
||||
and cos.is_contiguous()
|
||||
and sin.is_contiguous()
|
||||
for (cos, sin), expected_shape in zip(tables, expected_shapes, strict=True)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"can_use_ltx25_decoder_rope",
|
||||
"fused_ltx25_decoder_rope",
|
||||
]
|
||||
|
||||
@@ -33,7 +33,6 @@ if TYPE_CHECKING:
|
||||
from tvm_ffi.module import Module
|
||||
|
||||
_HEAD_DIM = 128
|
||||
_ROW_ALIGN = 8
|
||||
|
||||
|
||||
@cache_once
|
||||
@@ -50,117 +49,19 @@ def _jit_module(fuse_real_sin: bool, copy_v: bool) -> Module:
|
||||
)
|
||||
|
||||
|
||||
def _is_rows(t: torch.Tensor, device: torch.device, tokens: int | None = None) -> bool:
|
||||
"""``[B, T, H, 128]`` bf16 CUDA tensor whose heads are contiguous; batch/token strides may be padded."""
|
||||
return (
|
||||
isinstance(t, torch.Tensor)
|
||||
and t.dtype is torch.bfloat16
|
||||
and t.is_cuda
|
||||
and t.device == device
|
||||
and t.dim() == 4
|
||||
and t.numel() > 0
|
||||
and t.shape[-1] == _HEAD_DIM
|
||||
and (tokens is None or t.shape[1] == tokens)
|
||||
and t.stride(-1) == 1
|
||||
and t.stride(-2) == _HEAD_DIM
|
||||
and t.stride(1) % (_ROW_ALIGN // 2) == 0
|
||||
and t.stride(0) % (_ROW_ALIGN // 2) == 0
|
||||
and t.data_ptr() % _ROW_ALIGN == 0
|
||||
)
|
||||
|
||||
|
||||
def _is_weight(w: torch.Tensor, device: torch.device) -> bool:
|
||||
return (
|
||||
isinstance(w, torch.Tensor)
|
||||
and w.dtype is torch.bfloat16
|
||||
and w.is_cuda
|
||||
and w.device == device
|
||||
and w.shape == (_HEAD_DIM,)
|
||||
and w.is_contiguous()
|
||||
)
|
||||
|
||||
|
||||
def _is_rope(rope: torch.Tensor, seq: int, device: torch.device) -> bool:
|
||||
return (
|
||||
isinstance(rope, torch.Tensor)
|
||||
and rope.dtype is torch.complex64
|
||||
and rope.device == device
|
||||
and rope.shape == (seq, _HEAD_DIM // 2)
|
||||
and rope.is_contiguous()
|
||||
)
|
||||
|
||||
|
||||
def _cuda_platform(t: torch.Tensor) -> bool:
|
||||
return t.is_cuda and torch.version.hip is None and not torch.compiler.is_compiling()
|
||||
@torch.compiler.assume_constant_result
|
||||
@cache_once
|
||||
def can_use_qknorm_complex_rope_cuda(dtype: torch.dtype, head_dim: int) -> bool:
|
||||
"""The CUDA reduction is specialized to BF16, 128-wide heads."""
|
||||
return dtype is torch.bfloat16 and head_dim == _HEAD_DIM
|
||||
|
||||
|
||||
def _rope_real(rope: torch.Tensor) -> torch.Tensor:
|
||||
return torch.view_as_real(rope).view(rope.shape[0], _HEAD_DIM)
|
||||
|
||||
|
||||
def can_use_qknorm_complex_rope_cuda(
|
||||
x: torch.Tensor, weight: torch.Tensor, rope: torch.Tensor
|
||||
) -> bool:
|
||||
"""Contiguous ``[B, S, H, 128]`` bf16 Q or K with its ``(S, 64)`` complex64 table."""
|
||||
return (
|
||||
_cuda_platform(x)
|
||||
and _is_rows(x, x.device)
|
||||
and x.is_contiguous()
|
||||
and _is_weight(weight, x.device)
|
||||
and _is_rope(rope, x.shape[1], x.device)
|
||||
)
|
||||
|
||||
|
||||
def can_use_qknorm_complex_rope_pack(
|
||||
q: torch.Tensor,
|
||||
k_out: torch.Tensor,
|
||||
v_out: torch.Tensor,
|
||||
q_weight: torch.Tensor,
|
||||
k_weight: torch.Tensor,
|
||||
rope: torch.Tensor,
|
||||
k_prefix: torch.Tensor,
|
||||
v_prefix: torch.Tensor,
|
||||
k_src: Optional[torch.Tensor],
|
||||
v_src: Optional[torch.Tensor],
|
||||
) -> bool:
|
||||
"""Shapes for :func:`qknorm_complex_rope_pack_`; ``None`` sources mean "already in the buffer"."""
|
||||
if not (_cuda_platform(q) and _is_rows(q, q.device)):
|
||||
return False
|
||||
device = q.device
|
||||
batch, seq, heads, _ = q.shape
|
||||
if not (
|
||||
k_prefix.dim() == 4
|
||||
and k_prefix.shape[0] == batch
|
||||
and k_prefix.shape[1] > 0
|
||||
and k_prefix.shape[2] == heads
|
||||
and k_prefix.shape[3] == _HEAD_DIM
|
||||
and k_prefix.is_contiguous()
|
||||
and v_prefix.shape == k_prefix.shape
|
||||
and v_prefix.is_contiguous()
|
||||
and all(
|
||||
t.dtype is torch.bfloat16 and t.device == device
|
||||
for t in (k_prefix, v_prefix)
|
||||
)
|
||||
):
|
||||
return False
|
||||
prefix = k_prefix.shape[1]
|
||||
return (
|
||||
_is_rows(k_out, device, prefix + seq)
|
||||
and _is_rows(v_out, device, prefix + seq)
|
||||
and k_out.shape[0] == batch
|
||||
and v_out.shape[0] == batch
|
||||
and k_out.shape[2] == heads
|
||||
and v_out.shape[2] == heads
|
||||
and (k_src is None or (_is_rows(k_src, device, seq) and k_src.shape == q.shape))
|
||||
and (v_src is None or (_is_rows(v_src, device, seq) and v_src.shape == q.shape))
|
||||
and _is_weight(q_weight, device)
|
||||
and _is_weight(k_weight, device)
|
||||
and _is_rope(rope, seq, device)
|
||||
)
|
||||
return torch.view_as_real(rope)
|
||||
|
||||
|
||||
def _fake_norm_rope(x, weight, rope, eps):
|
||||
return torch.empty_like(x)
|
||||
return torch.empty(x.shape, dtype=x.dtype, device=x.device)
|
||||
|
||||
|
||||
@register_custom_op(
|
||||
@@ -171,8 +72,8 @@ def _fake_norm_rope(x, weight, rope, eps):
|
||||
def qknorm_complex_rope_cuda(
|
||||
x: torch.Tensor, weight: torch.Tensor, rope: torch.Tensor, eps: float
|
||||
) -> torch.Tensor:
|
||||
"""``rope(rmsnorm(x))`` for one contiguous ``[B, S, H, 128]`` tensor, out of place."""
|
||||
out = torch.empty_like(x)
|
||||
"""``rope(rmsnorm(x))`` for one ``[B, S, H, 128]`` tensor with packed head rows, out of place."""
|
||||
out = torch.empty(x.shape, dtype=x.dtype, device=x.device)
|
||||
module = _jit_module(_fuse_real_sin(x.device), False)
|
||||
with torch.cuda.device(x.device):
|
||||
module.norm_rope(x, out, weight, _rope_real(rope), float(eps))
|
||||
@@ -224,7 +125,6 @@ def qknorm_complex_rope_pack_(
|
||||
|
||||
__all__ = [
|
||||
"can_use_qknorm_complex_rope_cuda",
|
||||
"can_use_qknorm_complex_rope_pack",
|
||||
"qknorm_complex_rope_cuda",
|
||||
"qknorm_complex_rope_pack_",
|
||||
]
|
||||
|
||||
@@ -8,7 +8,7 @@ import triton.language as tl
|
||||
from sglang.kernels.ops.diffusion.rope.complex_rope_triton import _fuse_real_sin
|
||||
from sglang.kernels.ops.diffusion.rope.qknorm_complex_rope_triton import (
|
||||
_qknorm_complex_rope_rows,
|
||||
can_use_qknorm_complex_rope,
|
||||
_validate_qknorm_complex_rope,
|
||||
)
|
||||
from sglang.srt.utils.custom_op import register_custom_op
|
||||
|
||||
@@ -54,22 +54,6 @@ def _qknorm_complex_rope_kv_kernel(
|
||||
tl.store(vout_ptr + prefix_index, prefix_value, prefix_mask)
|
||||
|
||||
|
||||
def can_use_qknorm_complex_rope_kv(k, weight, rope, v, k_prefix, v_prefix):
|
||||
return (
|
||||
can_use_qknorm_complex_rope(k, weight, rope)
|
||||
and v.shape == k.shape
|
||||
and k_prefix.ndim == 4
|
||||
and k_prefix.shape[0] == k.shape[0]
|
||||
and k_prefix.shape[1] > 0
|
||||
and k_prefix.shape[2:] == k.shape[2:]
|
||||
and v_prefix.shape == k_prefix.shape
|
||||
and all(
|
||||
x.device == k.device and x.dtype == k.dtype and x.is_contiguous()
|
||||
for x in (v, k_prefix, v_prefix)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _fake_qknorm_complex_rope_kv(k, weight, rope, v, k_prefix, v_prefix, eps):
|
||||
shape = (k.shape[0], k_prefix.shape[1] + k.shape[1], *k.shape[2:])
|
||||
return k.new_empty(shape), v.new_empty(shape)
|
||||
@@ -89,7 +73,21 @@ def qknorm_complex_rope_kv(
|
||||
v_prefix: torch.Tensor,
|
||||
eps: float,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
assert can_use_qknorm_complex_rope_kv(k, weight, rope, v, k_prefix, v_prefix)
|
||||
k, v = k.contiguous(), v.contiguous()
|
||||
_validate_qknorm_complex_rope(k, weight, rope)
|
||||
if not (
|
||||
v.shape == k.shape
|
||||
and k_prefix.ndim == 4
|
||||
and k_prefix.shape[0] == k.shape[0]
|
||||
and k_prefix.shape[1] > 0
|
||||
and k_prefix.shape[2:] == k.shape[2:]
|
||||
and v_prefix.shape == k_prefix.shape
|
||||
and all(
|
||||
x.device == k.device and x.dtype == k.dtype and x.is_contiguous()
|
||||
for x in (v, k_prefix, v_prefix)
|
||||
)
|
||||
):
|
||||
raise RuntimeError("invalid input for qknorm_complex_rope_kv")
|
||||
kout, vout = _fake_qknorm_complex_rope_kv(
|
||||
k, weight, rope, v, k_prefix, v_prefix, eps
|
||||
)
|
||||
|
||||
@@ -5,12 +5,8 @@ import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
from sglang.kernels.ops.diffusion.norm.rmsnorm_preserve_reduction import (
|
||||
can_use_rmsnorm_preserve_reduction,
|
||||
)
|
||||
from sglang.kernels.ops.diffusion.rope.complex_rope_triton import (
|
||||
_fuse_real_sin,
|
||||
can_use_fused_complex_rope,
|
||||
)
|
||||
from sglang.srt.utils.custom_op import register_custom_op
|
||||
|
||||
@@ -80,16 +76,29 @@ def _qknorm_complex_rope_onepass_kernel(
|
||||
)
|
||||
|
||||
|
||||
def can_use_qknorm_complex_rope(x, weight, rope):
|
||||
return (
|
||||
can_use_rmsnorm_preserve_reduction(x, weight)
|
||||
and can_use_fused_complex_rope(x, rope)
|
||||
def _validate_qknorm_complex_rope(x, weight, rope):
|
||||
if not (
|
||||
x.is_cuda
|
||||
and torch.version.hip is None
|
||||
and x.dtype in (torch.float16, torch.bfloat16)
|
||||
and x.ndim == 4
|
||||
and x.shape[-1] == 128
|
||||
)
|
||||
and x.numel() > 0
|
||||
and x.is_contiguous()
|
||||
and weight.device == x.device
|
||||
and weight.dtype == x.dtype
|
||||
and weight.shape == (128,)
|
||||
and weight.is_contiguous()
|
||||
and rope.dtype == torch.complex64
|
||||
and rope.device == x.device
|
||||
and rope.shape == (x.shape[1], 64)
|
||||
and rope.is_contiguous()
|
||||
):
|
||||
raise RuntimeError("invalid input for qknorm_complex_rope")
|
||||
|
||||
|
||||
def _fake_qknorm_complex_rope(x, weight, rope, eps):
|
||||
return torch.empty_like(x)
|
||||
return torch.empty(x.shape, dtype=x.dtype, device=x.device)
|
||||
|
||||
|
||||
@register_custom_op(
|
||||
@@ -103,7 +112,8 @@ def qknorm_complex_rope(
|
||||
rope: torch.Tensor,
|
||||
eps: float,
|
||||
) -> torch.Tensor:
|
||||
assert can_use_qknorm_complex_rope(x, weight, rope)
|
||||
x = x.contiguous()
|
||||
_validate_qknorm_complex_rope(x, weight, rope)
|
||||
out = torch.empty_like(x)
|
||||
with torch.cuda.device(x.device):
|
||||
_qknorm_complex_rope_onepass_kernel[(triton.cdiv(x.numel() // 128, 4),)](
|
||||
|
||||
@@ -83,32 +83,6 @@ def _rope_rotate_half_kernel(
|
||||
tl.store(out_ptr + toff, tail, mask=tmask)
|
||||
|
||||
|
||||
def can_use_fused_rope_rotate_half(
|
||||
x: torch.Tensor,
|
||||
cos: torch.Tensor,
|
||||
sin: torch.Tensor,
|
||||
) -> bool:
|
||||
if x.dtype is not torch.bfloat16 or not x.is_cuda:
|
||||
return False
|
||||
if x.dim() != 4 or not x.is_contiguous():
|
||||
return False
|
||||
rows = x.shape[0] * x.shape[1]
|
||||
rot = cos.shape[-1]
|
||||
return (
|
||||
cos.dtype is torch.bfloat16
|
||||
and sin.dtype is torch.bfloat16
|
||||
and cos.is_cuda
|
||||
and cos.device == x.device
|
||||
and sin.device == x.device
|
||||
and cos.shape == (rows, rot)
|
||||
and sin.shape == (rows, rot)
|
||||
and cos.is_contiguous()
|
||||
and sin.is_contiguous()
|
||||
and rot % 2 == 0
|
||||
and 0 < rot <= x.shape[-1]
|
||||
)
|
||||
|
||||
|
||||
def _fake_rope_rotate_half(
|
||||
x: torch.Tensor,
|
||||
cos: torch.Tensor,
|
||||
@@ -129,15 +103,36 @@ def fused_rope_rotate_half_bitexact(
|
||||
) -> torch.Tensor:
|
||||
"""Rotate-half RoPE over the leading ``cos.shape[-1]`` columns of ``x``.
|
||||
|
||||
``x`` is ``(B, S, H, D)``; ``cos``/``sin`` are ``(B * S, rot_dim)`` rows.
|
||||
``x`` is ``(B, S, H, D)``; ``cos``/``sin`` are contiguous
|
||||
``(B * S, rot_dim)`` rows or ``(B, S, 1, rot_dim)`` broadcast tables.
|
||||
Bit-exact vs the eager chunk/neg/cat/mul/add chain.
|
||||
"""
|
||||
if not (
|
||||
x.is_cuda and x.dtype is torch.bfloat16 and x.ndim == 4 and x.is_contiguous()
|
||||
):
|
||||
raise RuntimeError("rotate-half RoPE expects contiguous BF16 CUDA [B, S, H, D]")
|
||||
batch, seq_len, heads, head_dim = x.shape
|
||||
if cos.ndim not in (2, 4):
|
||||
raise RuntimeError("cos must have shape [B * S, rot_dim] or [B, S, 1, rot_dim]")
|
||||
rot = cos.shape[-1]
|
||||
if not (0 < rot <= head_dim and rot % 2 == 0):
|
||||
raise RuntimeError("rot_dim must be positive, even and no larger than head_dim")
|
||||
table_shape = (batch * seq_len, rot) if cos.ndim == 2 else (batch, seq_len, 1, rot)
|
||||
device = x.device
|
||||
for name, tensor in (("cos", cos), ("sin", sin)):
|
||||
if not (
|
||||
tensor.dtype is torch.bfloat16
|
||||
and tensor.device == device
|
||||
and tensor.shape == table_shape
|
||||
and tensor.is_contiguous()
|
||||
):
|
||||
raise RuntimeError(
|
||||
f"{name} must be contiguous {table_shape} with x's dtype/device"
|
||||
)
|
||||
half = rot // 2
|
||||
out = torch.empty_like(x)
|
||||
tail = head_dim - rot
|
||||
with torch.cuda.device(x.device):
|
||||
with torch.cuda.device(device):
|
||||
_rope_rotate_half_kernel[(batch * seq_len,)](
|
||||
out,
|
||||
x,
|
||||
|
||||
@@ -22,7 +22,6 @@ import torch
|
||||
from torch import nn
|
||||
|
||||
from sglang.kernels.ops.diffusion.norm.rmsnorm_scale_shift_bitexact import (
|
||||
can_use_fused_rmsnorm_scale_shift,
|
||||
fused_rmsnorm_scale_shift_bitexact,
|
||||
)
|
||||
from sglang.kernels.ops.diffusion.sites.quality_gate import QualityGatedFusion
|
||||
@@ -65,14 +64,6 @@ def _ones_weight(x: torch.Tensor) -> torch.Tensor:
|
||||
return w
|
||||
|
||||
|
||||
def can_use_ltx2_rms_norm_modulate(
|
||||
x: torch.Tensor, scale: torch.Tensor, shift: torch.Tensor
|
||||
) -> bool:
|
||||
if x.dtype is not torch.bfloat16 or not x.is_cuda:
|
||||
return False
|
||||
return can_use_fused_rmsnorm_scale_shift(x, _ones_weight(x), scale, shift)
|
||||
|
||||
|
||||
def fused_ltx2_rms_norm_modulate(
|
||||
x: torch.Tensor, scale: torch.Tensor, shift: torch.Tensor, eps: float
|
||||
) -> torch.Tensor:
|
||||
|
||||
@@ -19,7 +19,6 @@ from torch import nn
|
||||
|
||||
from sglang.kernels.ops.diffusion import (
|
||||
BitExactFusionGate,
|
||||
can_use_ltx25_decoder_rope,
|
||||
fused_ltx25_decoder_rope,
|
||||
tensors_equal,
|
||||
)
|
||||
@@ -32,8 +31,11 @@ from sglang.multimodal_gen.runtime.layers.visual_embedding import (
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import (
|
||||
LayerwiseOffloadableModuleMixin,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.platforms import current_platform
|
||||
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
|
||||
|
||||
_is_cuda = current_platform.is_cuda()
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# Bounds the three hidden-width temporaries the SwiGLU holds live. Exact: the
|
||||
@@ -326,7 +328,9 @@ class LTX2VideoVaeRotaryPosEmbed3D(nn.Module):
|
||||
verified = _LTX25_DECODER_ROPE.verified
|
||||
if (
|
||||
not _LTX25_DECODER_ROPE.disabled
|
||||
and can_use_ltx25_decoder_rope(query, key, tables, self.rope_dim_split)
|
||||
and _is_cuda
|
||||
and query.is_cuda
|
||||
and query.dtype is torch.bfloat16
|
||||
and (verified or _LTX25_DECODER_ROPE.can_attempt_once())
|
||||
):
|
||||
dim_t, dim_h, _ = self.rope_dim_split
|
||||
|
||||
@@ -25,8 +25,6 @@ from sglang.kernels.ops.activation.activation import (
|
||||
from sglang.kernels.ops.diffusion import (
|
||||
BitExactFusionGate,
|
||||
can_use_fused_rmsnorm_scale_shift,
|
||||
can_use_fused_rope_rotate_half,
|
||||
can_use_fused_scale_residual_rmsnorm_scale_shift,
|
||||
flashinfer_rmsnorm_diagnostic_hint,
|
||||
fused_rmsnorm_scale_shift_bitexact,
|
||||
fused_rope_rotate_half_bitexact,
|
||||
@@ -92,8 +90,9 @@ def _ernie_norm_scale_shift(
|
||||
verified = _ERNIE_NORM.verified
|
||||
if (
|
||||
not _ERNIE_NORM.disabled
|
||||
and x.is_cuda
|
||||
and norm.variance_size_override is None
|
||||
and can_use_fused_rmsnorm_scale_shift(x, norm.weight, scale, shift)
|
||||
and can_use_fused_rmsnorm_scale_shift(x.dtype, x.shape[-1])
|
||||
and (verified or _ERNIE_NORM.can_attempt_once())
|
||||
):
|
||||
try:
|
||||
@@ -136,10 +135,9 @@ def _ernie_gated_norm_scale_shift(
|
||||
verified = _ERNIE_GATED_NORM.verified
|
||||
if (
|
||||
not _ERNIE_GATED_NORM.disabled
|
||||
and residual.is_cuda
|
||||
and norm.variance_size_override is None
|
||||
and can_use_fused_scale_residual_rmsnorm_scale_shift(
|
||||
residual, update, gate, norm.weight, scale, shift
|
||||
)
|
||||
and can_use_fused_rmsnorm_scale_shift(residual.dtype, residual.shape[-1])
|
||||
and (verified or _ERNIE_GATED_NORM.can_attempt_once())
|
||||
):
|
||||
try:
|
||||
@@ -461,7 +459,8 @@ def _ernie_rope(
|
||||
verified = _ERNIE_ROPE.verified
|
||||
if (
|
||||
not _ERNIE_ROPE.disabled
|
||||
and can_use_fused_rope_rotate_half(x, cos_, sin_)
|
||||
and x.is_cuda
|
||||
and x.dtype is torch.bfloat16
|
||||
and (verified or _ERNIE_ROPE.can_attempt_once())
|
||||
):
|
||||
try:
|
||||
|
||||
@@ -122,8 +122,9 @@ def _flux_fused_ln_modulate(
|
||||
"""
|
||||
if (
|
||||
_FLUX_LN_MOD.disabled
|
||||
or not x.is_cuda
|
||||
or not is_plain_layer_norm(norm, x.shape[-1])
|
||||
or not can_use_fused_layernorm_modulate(x, scale, shift)
|
||||
or not can_use_fused_layernorm_modulate(x.dtype, x.shape[-1])
|
||||
):
|
||||
return None
|
||||
sig = (
|
||||
|
||||
@@ -153,7 +153,7 @@ def _norm_modulate(
|
||||
_LN_MODULATE.disabled
|
||||
or not _eager_fast_path_allowed(x)
|
||||
or not is_plain_layer_norm(norm, x.shape[-1])
|
||||
or not can_use_fused_layernorm_modulate(x, scale_row, shift_row)
|
||||
or not can_use_fused_layernorm_modulate(x.dtype, x.shape[-1])
|
||||
):
|
||||
return (1 + scale) * norm(x) + shift
|
||||
sig = (x.device, x.shape[0], x.shape[-1], norm.eps)
|
||||
|
||||
@@ -208,9 +208,10 @@ def _try_flux2_norm_modulate_fp8(
|
||||
shift_row = shift.squeeze(1) if shift.dim() == 3 and shift.shape[1] == 1 else shift
|
||||
if (
|
||||
_FLUX2_LN_FP8.disabled
|
||||
or not x.is_cuda
|
||||
or x.shape[-1] != 6144
|
||||
or not is_plain_layer_norm(norm, x.shape[-1])
|
||||
or not can_use_fused_layernorm_modulate(x, scale_row, shift_row)
|
||||
or not can_use_fused_layernorm_modulate(x.dtype, x.shape[-1])
|
||||
):
|
||||
return None
|
||||
|
||||
@@ -373,8 +374,9 @@ def _flux2_norm_modulate(
|
||||
shift_row = shift.squeeze(1) if shift.dim() == 3 and shift.shape[1] == 1 else shift
|
||||
if (
|
||||
_FLUX2_LN_MOD.disabled
|
||||
or not x.is_cuda
|
||||
or not is_plain_layer_norm(norm, x.shape[-1])
|
||||
or not can_use_fused_layernorm_modulate(x, scale_row, shift_row)
|
||||
or not can_use_fused_layernorm_modulate(x.dtype, x.shape[-1])
|
||||
):
|
||||
return norm(x) * (1 + scale) + shift
|
||||
|
||||
|
||||
@@ -103,9 +103,10 @@ def _glm_ln_modulate(
|
||||
if (
|
||||
not _GLM_LN_MOD.disabled
|
||||
and _is_cuda
|
||||
and x.is_cuda
|
||||
and dtype is x.dtype
|
||||
and is_plain_layer_norm(norm, x.shape[-1])
|
||||
and can_use_fused_layernorm_modulate(x, scale, shift)
|
||||
and can_use_fused_layernorm_modulate(x.dtype, x.shape[-1])
|
||||
and (verified or _GLM_LN_MOD.can_attempt_once())
|
||||
):
|
||||
try:
|
||||
@@ -144,12 +145,13 @@ def _glm_qk_layernorm(
|
||||
if (
|
||||
not _GLM_QK_LN.disabled
|
||||
and _is_cuda
|
||||
and query.is_cuda
|
||||
and dtype is query.dtype
|
||||
and dtype is key.dtype
|
||||
and is_plain_layer_norm(norm_q, query.shape[-1])
|
||||
and is_plain_layer_norm(norm_k, key.shape[-1])
|
||||
and norm_q.eps == norm_k.eps
|
||||
and can_use_fused_qk_head_layernorm(query, key)
|
||||
and can_use_fused_qk_head_layernorm(query.dtype, query.shape[-1])
|
||||
and (verified or _GLM_QK_LN.can_attempt_once())
|
||||
):
|
||||
try:
|
||||
|
||||
@@ -17,7 +17,6 @@ import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
|
||||
from sglang.kernels.ops.diffusion import (
|
||||
can_use_helios_qk_rope,
|
||||
fused_inplace_helios_qk_rope,
|
||||
mark_helios_gated_residual_site,
|
||||
try_helios_gated_residual,
|
||||
@@ -58,8 +57,11 @@ from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload im
|
||||
LayerwiseOffloadableModuleMixin,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.dits.base import CachableDiT
|
||||
from sglang.multimodal_gen.runtime.platforms import current_platform
|
||||
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
|
||||
|
||||
_is_cuda = current_platform.is_cuda()
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@@ -297,12 +299,8 @@ class HeliosSelfAttention(nn.Module):
|
||||
k: torch.Tensor,
|
||||
rotary_emb: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
if not self.tp_rmsnorm and can_use_helios_qk_rope(q, k, rotary_emb):
|
||||
fused_inplace_helios_qk_rope(
|
||||
q.view(-1, q.shape[-2], q.shape[-1]),
|
||||
k.view(-1, k.shape[-2], k.shape[-1]),
|
||||
rotary_emb.view(-1, rotary_emb.shape[-1]),
|
||||
)
|
||||
if not self.tp_rmsnorm and _is_cuda and q.is_cuda:
|
||||
fused_inplace_helios_qk_rope(q, k, rotary_emb)
|
||||
return q, k
|
||||
return (
|
||||
apply_rotary_emb_transposed(q, rotary_emb),
|
||||
|
||||
@@ -9,7 +9,6 @@ import torch.nn.functional as F
|
||||
|
||||
from sglang.kernels.ops.diffusion import (
|
||||
BitExactFusionGate,
|
||||
can_use_fused_silu_mul,
|
||||
fused_gate_rmsnorm_active,
|
||||
fused_rmsnorm_scale,
|
||||
fused_rmsnorm_tanh_residual,
|
||||
@@ -54,9 +53,14 @@ from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload im
|
||||
LayerwiseOffloadableModuleMixin,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.dits.base import BaseDiT
|
||||
from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum
|
||||
from sglang.multimodal_gen.runtime.platforms import (
|
||||
AttentionBackendEnum,
|
||||
current_platform,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
|
||||
|
||||
_is_cuda = current_platform.is_cuda()
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
OUTPUT_IMAGE_INDICATOR = 2
|
||||
@@ -67,35 +71,6 @@ _IDEOGRAM_SWIGLU = BitExactFusionGate("Ideogram fused SiLU-mul")
|
||||
_IDEOGRAM_ZERO_SHIFTS: dict[tuple[torch.device, torch.dtype, int], torch.Tensor] = {}
|
||||
|
||||
|
||||
def _can_use_fused_rope(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
cos: torch.Tensor,
|
||||
sin: torch.Tensor,
|
||||
) -> bool:
|
||||
# cos/sin are full-span (B, S, 1, D) rows broadcast over heads.
|
||||
expected = (q.shape[0], q.shape[1], 1, q.shape[-1])
|
||||
return (
|
||||
q.dtype is torch.bfloat16
|
||||
and k.dtype is torch.bfloat16
|
||||
and q.is_cuda
|
||||
and q.dim() == 4
|
||||
and q.is_contiguous()
|
||||
and k.shape == q.shape
|
||||
and k.is_contiguous()
|
||||
and k.device == q.device
|
||||
and cos.dtype is torch.bfloat16
|
||||
and sin.dtype is torch.bfloat16
|
||||
and cos.device == q.device
|
||||
and sin.device == q.device
|
||||
and cos.shape == expected
|
||||
and sin.shape == expected
|
||||
and cos.is_contiguous()
|
||||
and sin.is_contiguous()
|
||||
and q.shape[-1] % 2 == 0
|
||||
)
|
||||
|
||||
|
||||
def _ideogram_rope(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
@@ -113,14 +88,15 @@ def _ideogram_rope(
|
||||
verified = _IDEOGRAM_ROPE.verified
|
||||
if (
|
||||
not _IDEOGRAM_ROPE.disabled
|
||||
and _can_use_fused_rope(q, k, cos, sin)
|
||||
and q.is_cuda
|
||||
and q.dtype is torch.bfloat16
|
||||
and q.is_contiguous()
|
||||
and k.is_contiguous()
|
||||
and (verified or _IDEOGRAM_ROPE.can_attempt_once())
|
||||
):
|
||||
try:
|
||||
cos_rows = cos.reshape(-1, cos.shape[-1])
|
||||
sin_rows = sin.reshape(-1, sin.shape[-1])
|
||||
q_fused = fused_rope_rotate_half_bitexact(q, cos_rows, sin_rows)
|
||||
k_fused = fused_rope_rotate_half_bitexact(k, cos_rows, sin_rows)
|
||||
q_fused = fused_rope_rotate_half_bitexact(q, cos, sin)
|
||||
k_fused = fused_rope_rotate_half_bitexact(k, cos, sin)
|
||||
except Exception as exc:
|
||||
_IDEOGRAM_ROPE.on_exception(exc, logger=logger)
|
||||
else:
|
||||
@@ -144,7 +120,9 @@ def _ideogram_swiglu(a: torch.Tensor, b: torch.Tensor) -> torch.Tensor:
|
||||
verified = _IDEOGRAM_SWIGLU.verified
|
||||
if (
|
||||
not _IDEOGRAM_SWIGLU.disabled
|
||||
and can_use_fused_silu_mul(a, b)
|
||||
and _is_cuda
|
||||
and a.is_cuda
|
||||
and a.dtype is torch.bfloat16
|
||||
and (verified or _IDEOGRAM_SWIGLU.can_attempt_once())
|
||||
):
|
||||
try:
|
||||
|
||||
@@ -177,7 +177,9 @@ def _joy_joint_qkv(*inputs: torch.Tensor) -> tuple[torch.Tensor, ...]:
|
||||
or not inputs[0].is_cuda
|
||||
or torch.version.hip
|
||||
or torch.compiler.is_compiling()
|
||||
or not diffusion_kernels.can_use_joint_qkv_cat(*inputs)
|
||||
or inputs[0].dtype not in (torch.float16, torch.bfloat16)
|
||||
or inputs[0].shape[-2] * inputs[0].shape[-1] > 8192
|
||||
or any(x.requires_grad for x in inputs)
|
||||
):
|
||||
return reference()
|
||||
sig = (inputs[0].device, inputs[0].dtype) + tuple(
|
||||
|
||||
@@ -58,8 +58,11 @@ from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload im
|
||||
LayerwiseOffloadableModuleMixin,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.dits.base import BaseDiT
|
||||
from sglang.multimodal_gen.runtime.platforms import current_platform
|
||||
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
|
||||
|
||||
_is_cuda = current_platform.is_cuda()
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_LONGCAT_QKNORM_ROPE = BitExactFusionGate("LongCat fused QKNorm+RoPE")
|
||||
@@ -74,7 +77,11 @@ def _longcat_gelu_cat(
|
||||
type(activation) is nn.GELU
|
||||
and activation.approximate == "tanh"
|
||||
and _LONGCAT_GELU_CAT.can_attempt_once()
|
||||
and diffusion_ops.can_use_fused_gelu_tanh_cat(attn, mlp)
|
||||
and _is_cuda
|
||||
and attn.is_cuda
|
||||
and attn.dtype is torch.bfloat16
|
||||
and not torch.is_grad_enabled()
|
||||
and not torch.compiler.is_compiling()
|
||||
):
|
||||
try:
|
||||
output = diffusion_ops.fused_gelu_tanh_cat(attn, mlp)
|
||||
@@ -104,7 +111,7 @@ def _longcat_norm_modulate(
|
||||
not _LONGCAT_LN_MOD.disabled
|
||||
and x.is_cuda
|
||||
and diffusion_ops.is_plain_layer_norm(norm, x.shape[-1])
|
||||
and diffusion_ops.can_use_fused_layernorm_modulate(x, scale, shift)
|
||||
and diffusion_ops.can_use_fused_layernorm_modulate(x.dtype, x.shape[-1])
|
||||
):
|
||||
sig = (
|
||||
x.shape,
|
||||
|
||||
@@ -12,9 +12,9 @@ import torch.nn.functional as F
|
||||
|
||||
from sglang.kernels.ops.diffusion import (
|
||||
BitExactFusionGate,
|
||||
can_use_fused_rmsnorm_scale_shift,
|
||||
can_use_linear_gelu,
|
||||
can_use_ltx2_qknorm_split_rope_cuda,
|
||||
can_use_ltx2_rms_norm_modulate,
|
||||
can_use_modulate_scale_shift_cuda,
|
||||
fused_gelu_active,
|
||||
fused_linear_gelu_tanh,
|
||||
@@ -208,8 +208,10 @@ def _ltx2_rms_norm_modulate(
|
||||
default). The fused kernel is not bit-exact (<=1 bf16 ULP) so it is gated
|
||||
on the request-scoped mount rather than a runtime self-check.
|
||||
"""
|
||||
if ltx2_rms_norm_modulate_active(block) and can_use_ltx2_rms_norm_modulate(
|
||||
x, scale, shift
|
||||
if (
|
||||
ltx2_rms_norm_modulate_active(block)
|
||||
and x.is_cuda
|
||||
and can_use_fused_rmsnorm_scale_shift(x.dtype, x.shape[-1])
|
||||
):
|
||||
return fused_ltx2_rms_norm_modulate(x, scale, shift, eps)
|
||||
normed = rms_norm(x, eps)
|
||||
|
||||
@@ -520,7 +520,7 @@ def delta_factor_apply(
|
||||
B32.contiguous(),
|
||||
alpha.float().contiguous(),
|
||||
)
|
||||
if can_use_vdn_delta_factors(A32, B32, alpha32):
|
||||
if can_use_vdn_delta_factors(A32.device, A32.dtype, A32.shape[-1]):
|
||||
return vdn_delta_factors(A32, B32, alpha32)
|
||||
chol = torch.linalg.cholesky(A32 + eye)
|
||||
# (I+A)^-1 = L^-T L^-1: a batched trsm at 128x128 is far slower than the GEMM
|
||||
|
||||
@@ -133,8 +133,11 @@ def _qwen_norm_out(
|
||||
scale, shift = torch.chunk(emb, 2, dim=1)
|
||||
if (
|
||||
_QWEN_NORM_OUT.disabled
|
||||
or not hidden_states.is_cuda
|
||||
or not is_plain_layer_norm(norm_out.norm, hidden_states.shape[-1])
|
||||
or not can_use_fused_layernorm_modulate(hidden_states, scale, shift)
|
||||
or not can_use_fused_layernorm_modulate(
|
||||
hidden_states.dtype, hidden_states.shape[-1]
|
||||
)
|
||||
):
|
||||
return (
|
||||
norm_out.norm(hidden_states) * (1 + scale)[:, None, :] + shift[:, None, :]
|
||||
|
||||
@@ -8,12 +8,8 @@ from torch import nn
|
||||
|
||||
from sglang.kernels.ops.diffusion import (
|
||||
BitExactFusionGate,
|
||||
can_use_fused_complex_rope,
|
||||
can_use_fused_layernorm_modulate,
|
||||
can_use_fused_silu_mul,
|
||||
can_use_qknorm_complex_rope_cuda,
|
||||
can_use_qknorm_complex_rope_pack,
|
||||
can_use_rmsnorm_preserve_reduction,
|
||||
fused_complex_rope,
|
||||
fused_layernorm_modulate,
|
||||
fused_silu_mul_bitexact,
|
||||
@@ -24,11 +20,9 @@ from sglang.kernels.ops.diffusion import (
|
||||
tensors_equal,
|
||||
)
|
||||
from sglang.kernels.ops.diffusion.rope.qknorm_complex_rope_kv_triton import (
|
||||
can_use_qknorm_complex_rope_kv,
|
||||
qknorm_complex_rope_kv,
|
||||
)
|
||||
from sglang.kernels.ops.diffusion.rope.qknorm_complex_rope_triton import (
|
||||
can_use_qknorm_complex_rope,
|
||||
qknorm_complex_rope,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.distributed import (
|
||||
@@ -51,10 +45,15 @@ from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload im
|
||||
LayerwiseOffloadableModuleMixin,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.dits.base import CachableDiT
|
||||
from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum
|
||||
from sglang.multimodal_gen.runtime.platforms import (
|
||||
AttentionBackendEnum,
|
||||
current_platform,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
|
||||
_is_cuda = current_platform.is_cuda()
|
||||
|
||||
logger = init_logger(__name__)
|
||||
_ROPE_FUSION = BitExactFusionGate("Qwen-Image 2.1 complex RoPE")
|
||||
_SILU_MUL_FUSION = BitExactFusionGate("Qwen-Image 2.1 SiLU-mul")
|
||||
@@ -130,7 +129,7 @@ def build_layout(image_slots, image_shapes, axes_dims, device):
|
||||
|
||||
def apply_rope(x, rope):
|
||||
fused = None
|
||||
if can_use_fused_complex_rope(x, rope) and _ROPE_FUSION.can_attempt_once():
|
||||
if _is_cuda and x.is_cuda and _ROPE_FUSION.can_attempt_once():
|
||||
fused = fused_complex_rope(x, rope)
|
||||
if _ROPE_FUSION.verified:
|
||||
return fused
|
||||
@@ -144,7 +143,9 @@ def apply_rope(x, rope):
|
||||
def apply_qk_norm(x, norm):
|
||||
fused = None
|
||||
if (
|
||||
can_use_rmsnorm_preserve_reduction(x, norm.weight)
|
||||
_is_cuda
|
||||
and x.is_cuda
|
||||
and x.dtype in (torch.float16, torch.bfloat16)
|
||||
and _QK_NORM_FUSION.can_attempt_once()
|
||||
):
|
||||
fused = rmsnorm_preserve_reduction(x, norm.weight, norm.variance_epsilon)
|
||||
@@ -159,22 +160,28 @@ def apply_qk_norm(x, norm):
|
||||
def apply_qk_norm_rope(x, norm, rope):
|
||||
fused = None
|
||||
if (
|
||||
can_use_qknorm_complex_rope_cuda(x, norm.weight, rope)
|
||||
_is_cuda
|
||||
and x.is_cuda
|
||||
and not torch.compiler.is_compiling()
|
||||
and can_use_qknorm_complex_rope_cuda(x.dtype, x.shape[-1])
|
||||
and _QK_ROPE_CUDA_FUSION.can_attempt_once()
|
||||
):
|
||||
fused = qknorm_complex_rope_cuda(x, norm.weight, rope, norm.variance_epsilon)
|
||||
if _QK_ROPE_CUDA_FUSION.verified:
|
||||
return fused
|
||||
out = apply_rope(apply_qk_norm(x, norm), rope)
|
||||
out = apply_rope(norm(x), rope)
|
||||
return _QK_ROPE_CUDA_FUSION.accept_or_fallback(fused, out, logger=logger)
|
||||
if (
|
||||
can_use_qknorm_complex_rope(x, norm.weight, rope)
|
||||
_is_cuda
|
||||
and x.is_cuda
|
||||
and x.dtype in (torch.float16, torch.bfloat16)
|
||||
and x.shape[-1] == 128
|
||||
and _QK_ROPE_FUSION.can_attempt_once()
|
||||
):
|
||||
fused = qknorm_complex_rope(x, norm.weight, rope, norm.variance_epsilon)
|
||||
if _QK_ROPE_FUSION.verified:
|
||||
return fused
|
||||
out = apply_rope(apply_qk_norm(x, norm), rope)
|
||||
out = apply_rope(norm(x), rope)
|
||||
if fused is not None:
|
||||
return _QK_ROPE_FUSION.accept_or_fallback(fused, out, logger=logger)
|
||||
return out
|
||||
@@ -218,7 +225,8 @@ def cat_outputs(outputs, dim=0):
|
||||
def apply_modulation(x, norm, scale):
|
||||
fused = None
|
||||
if (
|
||||
can_use_fused_layernorm_modulate(x, scale.squeeze(1), None)
|
||||
x.is_cuda
|
||||
and can_use_fused_layernorm_modulate(x.dtype, x.shape[-1])
|
||||
and _MODULATION_FUSION.can_attempt_once()
|
||||
):
|
||||
fused = fused_layernorm_modulate(x, scale.squeeze(1), None, norm.eps)
|
||||
@@ -305,7 +313,11 @@ class QwenImage21FeedForward(nn.Module):
|
||||
def forward(self, x):
|
||||
gate, value = self.gate_layer(x)[0], self.proj(x)[0]
|
||||
fused = None
|
||||
if can_use_fused_silu_mul(gate, value) and _SILU_MUL_FUSION.can_attempt_once():
|
||||
if (
|
||||
_is_cuda
|
||||
and gate.dtype is torch.bfloat16
|
||||
and _SILU_MUL_FUSION.can_attempt_once()
|
||||
):
|
||||
fused = fused_silu_mul_bitexact(gate, value)
|
||||
if _SILU_MUL_FUSION.verified:
|
||||
return self.out(fused)[0]
|
||||
@@ -443,9 +455,16 @@ class QwenImage21Attention(nn.Module):
|
||||
With ``k_out``/``v_out`` the projection already wrote the raw K/V into
|
||||
rows ``[P:]`` (``k``/``v`` are those views), so K is normalized in place
|
||||
and V is not copied. Returns ``(q, k_out, v_out)`` or ``None`` when the
|
||||
layout is unsupported. The first call also runs the eager chain and
|
||||
dtype/head dimension is unsupported. The first call also runs the eager chain and
|
||||
keeps its result on a mismatch.
|
||||
"""
|
||||
if not (
|
||||
_is_cuda
|
||||
and q.is_cuda
|
||||
and not torch.compiler.is_compiling()
|
||||
and can_use_qknorm_complex_rope_cuda(q.dtype, self.head_dim)
|
||||
):
|
||||
return None
|
||||
eps = self.norm_k.variance_epsilon
|
||||
if self.norm_q.variance_epsilon != eps:
|
||||
return None
|
||||
@@ -456,24 +475,11 @@ class QwenImage21Attention(nn.Module):
|
||||
k_src, v_src = k, v
|
||||
else:
|
||||
k_src, v_src = None, None
|
||||
if not can_use_qknorm_complex_rope_pack(
|
||||
q,
|
||||
k_out,
|
||||
v_out,
|
||||
self.norm_q.weight,
|
||||
self.norm_k.weight,
|
||||
rope,
|
||||
kp,
|
||||
vp,
|
||||
k_src,
|
||||
v_src,
|
||||
):
|
||||
return None
|
||||
reference = None
|
||||
if not _KV_PACK_CUDA_FUSION.verified:
|
||||
reference = (
|
||||
apply_rope(apply_qk_norm(q, self.norm_q), rope),
|
||||
torch.cat([kp, apply_rope(apply_qk_norm(k, self.norm_k), rope)], 1),
|
||||
apply_rope(self.norm_q(q), rope),
|
||||
torch.cat([kp, apply_rope(self.norm_k(k), rope)], 1),
|
||||
torch.cat([vp, v], 1),
|
||||
)
|
||||
qknorm_complex_rope_pack_(
|
||||
@@ -612,7 +618,10 @@ class QwenImage21Attention(nn.Module):
|
||||
packed = None
|
||||
if (
|
||||
get_sp_world_size() == 1
|
||||
and can_use_qknorm_complex_rope_kv(k, self.norm_k.weight, rope, v, kp, vp)
|
||||
and _is_cuda
|
||||
and k.is_cuda
|
||||
and k.dtype in (torch.float16, torch.bfloat16)
|
||||
and self.head_dim == 128
|
||||
and _KV_ROPE_FUSION.can_attempt_once()
|
||||
):
|
||||
packed = qknorm_complex_rope_kv(
|
||||
@@ -620,7 +629,7 @@ class QwenImage21Attention(nn.Module):
|
||||
)
|
||||
if not _KV_ROPE_FUSION.verified:
|
||||
reference = (
|
||||
torch.cat([kp, apply_rope(apply_qk_norm(k, self.norm_k), rope)], 1),
|
||||
torch.cat([kp, apply_rope(self.norm_k(k), rope)], 1),
|
||||
torch.cat([vp, v], 1),
|
||||
)
|
||||
packed = _KV_ROPE_FUSION.accept_or_fallback(
|
||||
|
||||
@@ -7,8 +7,6 @@ from diffusers.models.embeddings import PixArtAlphaTextProjection, TimestepEmbed
|
||||
|
||||
from sglang.kernels.ops.diffusion import (
|
||||
BitExactFusionGate,
|
||||
can_use_fused_bias_glu,
|
||||
can_use_fused_bias_silu,
|
||||
can_use_fused_layernorm_modulate,
|
||||
fused_bias_glu,
|
||||
fused_bias_silu,
|
||||
@@ -106,7 +104,7 @@ def sana_ln_modulate(
|
||||
and is_plain_layer_norm(norm, x.shape[-1])
|
||||
):
|
||||
x_c = x.contiguous()
|
||||
if not can_use_fused_layernorm_modulate(x_c, scale[:, 0], shift[:, 0]):
|
||||
if not can_use_fused_layernorm_modulate(x_c.dtype, x_c.shape[-1]):
|
||||
return _eager_ln_modulate(norm, x, scale, shift)
|
||||
try:
|
||||
out = fused_layernorm_modulate_raw(x_c, scale[:, 0], shift[:, 0], norm.eps)
|
||||
@@ -185,7 +183,7 @@ def sana_conv_bias_silu(
|
||||
return F.silu(_mps_safe_conv2d(conv, x))
|
||||
|
||||
raw = _conv2d_without_bias(conv, x)
|
||||
if not can_use_fused_bias_silu(raw, conv.bias):
|
||||
if raw.dtype is not torch.bfloat16:
|
||||
return F.silu(raw + conv.bias[None, :, None, None])
|
||||
verified = _SANA_CONV_SILU.verified
|
||||
if not verified and not _SANA_CONV_SILU.can_attempt_once():
|
||||
@@ -217,10 +215,8 @@ def sana_conv_bias_glu(
|
||||
return hidden_states * F.silu(gate)
|
||||
|
||||
raw = _conv2d_without_bias(conv, x)
|
||||
if not can_use_fused_bias_glu(raw, conv.bias):
|
||||
hidden_states, gate = torch.chunk(
|
||||
raw + conv.bias[None, :, None, None], 2, dim=1
|
||||
)
|
||||
if raw.dtype is not torch.bfloat16:
|
||||
hidden_states, gate = (raw + conv.bias[None, :, None, None]).chunk(2, dim=1)
|
||||
return hidden_states * F.silu(gate)
|
||||
verified = _SANA_CONV_GLU.verified
|
||||
if not verified and not _SANA_CONV_GLU.can_attempt_once():
|
||||
|
||||
@@ -11,7 +11,6 @@ from diffusers.models.embeddings import PixArtAlphaTextProjection
|
||||
|
||||
from sglang.kernels.ops.diffusion import (
|
||||
BitExactFusionGate,
|
||||
can_use_interleaved_rope_fp64,
|
||||
fused_interleaved_rope_fp64,
|
||||
mark_sana_video_linear_attention_site,
|
||||
tensors_equal,
|
||||
@@ -34,8 +33,11 @@ from sglang.multimodal_gen.runtime.models.dits.sana import (
|
||||
sana_ln_modulate,
|
||||
sana_residual_gate_add,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.platforms import current_platform
|
||||
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
|
||||
|
||||
_is_cuda = current_platform.is_cuda()
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_SANA_VIDEO_ROPE = BitExactFusionGate("SANA-Video fused RoPE")
|
||||
@@ -66,7 +68,9 @@ def apply_interleaved_rotary_emb_pair(
|
||||
verified = _SANA_VIDEO_ROPE.verified
|
||||
if (
|
||||
not _SANA_VIDEO_ROPE.disabled
|
||||
and can_use_interleaved_rope_fp64(query, key, freqs_cos, freqs_sin)
|
||||
and _is_cuda
|
||||
and query.is_cuda
|
||||
and query.dtype is torch.bfloat16
|
||||
and (verified or _SANA_VIDEO_ROPE.can_attempt_once())
|
||||
):
|
||||
try:
|
||||
|
||||
@@ -11,7 +11,6 @@ import torch.nn as nn
|
||||
|
||||
from sglang.kernels.ops.diffusion import (
|
||||
BitExactFusionGate,
|
||||
can_use_fused_temb_table_slices,
|
||||
can_use_linear_gelu,
|
||||
fused_gelu_active,
|
||||
fused_linear_gelu_tanh,
|
||||
@@ -410,7 +409,8 @@ def _wan_temb_table_slices(
|
||||
verified = _WAN_TEMB_SLICES.verified
|
||||
if (
|
||||
not _WAN_TEMB_SLICES.disabled
|
||||
and can_use_fused_temb_table_slices(table, temb)
|
||||
and _is_cuda
|
||||
and temb.is_cuda
|
||||
and (verified or _WAN_TEMB_SLICES.can_attempt_once())
|
||||
):
|
||||
try:
|
||||
|
||||
@@ -576,17 +576,20 @@ def materialize_overlay_model(
|
||||
overlay_repo_id = str(overlay_spec["overlay_repo_id"])
|
||||
overlay_revision = str(overlay_spec.get("overlay_revision", "main"))
|
||||
overlay_fingerprint = _compute_overlay_fingerprint(overlay_dir)
|
||||
source_revision = source_revision or _overlay_source_revision(
|
||||
overlay_spec, manifest
|
||||
)
|
||||
cache_identity = {
|
||||
"source_model_id": source_model_id,
|
||||
"overlay_repo_id": overlay_repo_id,
|
||||
"overlay_revision": overlay_revision,
|
||||
"materializer_version": materializer_version,
|
||||
"overlay_fingerprint": overlay_fingerprint,
|
||||
}
|
||||
if source_revision is not None:
|
||||
cache_identity["source_revision"] = source_revision
|
||||
cache_key = hashlib.sha256(
|
||||
json.dumps(
|
||||
{
|
||||
"source_model_id": source_model_id,
|
||||
"overlay_repo_id": overlay_repo_id,
|
||||
"overlay_revision": overlay_revision,
|
||||
"materializer_version": materializer_version,
|
||||
"overlay_fingerprint": overlay_fingerprint,
|
||||
},
|
||||
sort_keys=True,
|
||||
).encode("utf-8")
|
||||
json.dumps(cache_identity, sort_keys=True).encode("utf-8")
|
||||
).hexdigest()[:16]
|
||||
cache_root = os.path.join(get_diffusion_cache_root(), "materialized_models")
|
||||
_ensure_dir(cache_root)
|
||||
|
||||
@@ -449,8 +449,19 @@ def _run_file_suite(args, target_dir: Path) -> int:
|
||||
print(f"No valid test files found for suite '{args.suite}'.")
|
||||
return 1 if args.suite in STRICT_SUITES else 0
|
||||
|
||||
my_files = partition_items_by_index(
|
||||
suite_files_abs, args.partition_id, args.total_partitions
|
||||
)
|
||||
print(
|
||||
f"Suite: {args.suite} | Partition: {args.partition_id}/{args.total_partitions} "
|
||||
f"| Files: {len(my_files)}/{len(suite_files_abs)}",
|
||||
flush=True,
|
||||
)
|
||||
if not my_files:
|
||||
return 0
|
||||
|
||||
exit_code, _, _ = run_pytest(
|
||||
suite_files_abs,
|
||||
my_files,
|
||||
filter_expr=args.filter,
|
||||
junit_xml_path=None,
|
||||
)
|
||||
|
||||
@@ -356,10 +356,13 @@ class _Stage:
|
||||
["--component-residency", "all=component-offload"],
|
||||
"component-offload",
|
||||
),
|
||||
(
|
||||
pytest.param(
|
||||
"text_encoder_2",
|
||||
["--component-residency", "text_encoder=snapshot-offload"],
|
||||
"snapshot-offload",
|
||||
marks=pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="snapshot-offload requires CUDA"
|
||||
),
|
||||
),
|
||||
("text_encoder_2", ["--text-encoder-cpu-offload", "true"], "component-offload"),
|
||||
("text_encoder_2", ["--text-encoder-cpu-offload", "false"], "resident"),
|
||||
|
||||
@@ -24,6 +24,7 @@ from sglang.multimodal_gen.test.server.testcase_configs import (
|
||||
@pytest.fixture
|
||||
def validator(monkeypatch):
|
||||
monkeypatch.setenv("SGLANG_GEN_BASELINE", "0")
|
||||
monkeypatch.setattr(utils.current_platform, "is_hip", lambda: False)
|
||||
scenario = ScenarioConfig.from_dict(
|
||||
{
|
||||
"stages_ms": {},
|
||||
|
||||
@@ -1,6 +1,11 @@
|
||||
"""Unit tests for diffusion model-overlay cache paths and materialized trees."""
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from filelock import FileLock
|
||||
|
||||
from sglang.multimodal_gen.runtime.utils import model_overlay
|
||||
from sglang.multimodal_gen.runtime.utils.model_overlay import (
|
||||
@@ -65,6 +70,80 @@ def test_large_files_are_still_shared_with_the_cache(tmp_path, monkeypatch):
|
||||
assert os.path.samefile(out / "model.safetensors", weights)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("repair_cache", [False, True])
|
||||
def test_overlay_uses_pinned_source_weights(tmp_path, monkeypatch, repair_cache):
|
||||
overlay = tmp_path / "overlay"
|
||||
(overlay / "_overlay").mkdir(parents=True)
|
||||
(overlay / "_overlay" / "overlay_manifest.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"required_source_files": ["model.safetensors"],
|
||||
"file_mappings": [
|
||||
{"src": "model.safetensors", "dst": "transformer/model.safetensors"}
|
||||
],
|
||||
}
|
||||
)
|
||||
)
|
||||
(overlay / "model_index.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"_class_name": "TestPipeline",
|
||||
"_diffusers_version": "0.35.0",
|
||||
"transformer": ["diffusers", "Transformer"],
|
||||
}
|
||||
)
|
||||
)
|
||||
for revision in ("main", "original", "newer"):
|
||||
source = tmp_path / revision
|
||||
source.mkdir()
|
||||
if revision != "original" or not repair_cache:
|
||||
(source / "model.safetensors").write_text(revision)
|
||||
|
||||
spec = {"overlay_repo_id": "test/overlay", "source_revision": "original"}
|
||||
monkeypatch.setattr(
|
||||
model_overlay, "resolve_model_overlay_target", lambda _: ("test/model", spec)
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
model_overlay, "download_overlay_metadata", lambda *a, **kw: str(overlay)
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
model_overlay, "get_diffusion_cache_root", lambda: str(tmp_path / "cache")
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
model_overlay, "get_lock", lambda _: FileLock(tmp_path / "overlay.lock")
|
||||
)
|
||||
|
||||
def cached_snapshot(*args, revision=None, **kwargs):
|
||||
return str(tmp_path / (revision or "main"))
|
||||
|
||||
def download_snapshot(*args, revision=None, **kwargs):
|
||||
source = Path(cached_snapshot(revision=revision))
|
||||
(source / "model.safetensors").write_text(revision or "main")
|
||||
return str(source)
|
||||
|
||||
def resolve():
|
||||
return model_overlay.maybe_resolve_overlay_model_path(
|
||||
"test/model",
|
||||
local_dir=None,
|
||||
download=True,
|
||||
allow_patterns=None,
|
||||
snapshot_download_fn=download_snapshot,
|
||||
hf_hub_download_fn=None,
|
||||
verify_diffusers_model_complete_fn=lambda path: os.path.isfile(
|
||||
os.path.join(path, "model_index.json")
|
||||
),
|
||||
base_model_download_fn=cached_snapshot,
|
||||
)
|
||||
|
||||
original = Path(resolve())
|
||||
assert (original / "transformer/model.safetensors").read_text() == "original"
|
||||
# A new source revision must not reuse weights materialized from the old one.
|
||||
spec["source_revision"] = "newer"
|
||||
newer = Path(resolve())
|
||||
assert (newer / "transformer/model.safetensors").read_text() == "newer"
|
||||
assert original != newer
|
||||
|
||||
|
||||
def test_registry_source_revision_survives_upstream_removing_required_file(
|
||||
tmp_path, monkeypatch
|
||||
):
|
||||
|
||||
@@ -38,6 +38,7 @@ from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.qwen_image21 import (
|
||||
QwenImage21DenoisingStage,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.platforms import current_platform
|
||||
from sglang.multimodal_gen.runtime.server_args import (
|
||||
ServerArgs,
|
||||
set_global_server_args,
|
||||
@@ -46,7 +47,10 @@ from sglang.multimodal_gen.test.single_test_file.component_accuracy.utils import
|
||||
ensure_distributed_env_defaults,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not (torch.cuda.is_available() and current_platform.is_cuda()),
|
||||
reason="requires NVIDIA CUDA fusions",
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
|
||||
@@ -2,6 +2,7 @@ from dataclasses import replace
|
||||
|
||||
import pytest
|
||||
|
||||
from sglang.multimodal_gen.runtime.platforms import current_platform
|
||||
from sglang.multimodal_gen.test.server.test_server_utils import PerformanceValidator
|
||||
from sglang.multimodal_gen.test.server.testcase_configs import (
|
||||
BaselineConfig,
|
||||
@@ -69,6 +70,7 @@ def test_runner_override_preserves_other_metrics(monkeypatch, runner, flux, qwen
|
||||
],
|
||||
)
|
||||
def test_runner_baseline_enforces_e2e_boundary(monkeypatch, runner):
|
||||
monkeypatch.setattr(current_platform, "is_hip", lambda: False)
|
||||
monkeypatch.setenv("RUNNER_NAME", runner)
|
||||
monkeypatch.setenv("SGLANG_GEN_BASELINE", "0")
|
||||
monkeypatch.delenv("SGLANG_E2E_TOLERANCE", raising=False)
|
||||
|
||||
@@ -124,6 +124,7 @@ def harness(monkeypatch):
|
||||
monkeypatch.setenv("SGLANG_GEN_BASELINE", "0")
|
||||
monkeypatch.setenv("SGLANG_SKIP_CONSISTENCY", "0")
|
||||
monkeypatch.setattr(test_server_common.current_platform, "is_cuda", lambda: True)
|
||||
monkeypatch.setattr(test_server_common.current_platform, "is_hip", lambda: False)
|
||||
scenario = ScenarioConfig(
|
||||
stages_ms={"DenoisingStage": 10},
|
||||
denoise_step_ms={0: 5, 1: 5},
|
||||
|
||||
@@ -81,6 +81,34 @@ def test_suite_is_fully_scheduled_for_any_shard_count(suite, total_partitions):
|
||||
assert sorted(scheduled_standalone_files) == sorted(expected_standalone_files)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("total_partitions", [1, 4, 8])
|
||||
def test_file_suite_runs_each_file_once(monkeypatch, tmp_path, total_partitions):
|
||||
files = [f"test_{i}.py" for i in range(5)]
|
||||
for filename in files:
|
||||
(tmp_path / filename).touch()
|
||||
monkeypatch.setitem(run_suite.FILE_SUITES, "unit", files)
|
||||
executed = []
|
||||
|
||||
def run_files(assigned, filter_expr, junit_xml_path):
|
||||
assert assigned
|
||||
assert filter_expr == "not ltx2_vae_channels_last"
|
||||
executed.extend(assigned)
|
||||
return 1, [], {}
|
||||
|
||||
monkeypatch.setattr(run_suite, "run_pytest", run_files)
|
||||
for partition_id in range(total_partitions):
|
||||
args = SimpleNamespace(
|
||||
suite="unit",
|
||||
partition_id=partition_id,
|
||||
total_partitions=total_partitions,
|
||||
filter="not ltx2_vae_channels_last",
|
||||
)
|
||||
result = run_suite._run_file_suite(args, tmp_path)
|
||||
assert result == (1 if partition_id < len(files) else 0)
|
||||
|
||||
assert sorted(executed) == sorted(str(tmp_path / filename) for filename in files)
|
||||
|
||||
|
||||
def test_failing_cases_do_not_skip_the_shards_standalone_files(monkeypatch, tmp_path):
|
||||
"""Standalone files used to own a shard, so cases could not block them."""
|
||||
standalone_rel = "../single_test_file/test_disagg_server.py"
|
||||
|
||||
@@ -6,10 +6,14 @@ from sglang.multimodal_gen.runtime.models.dits.wanvideo import (
|
||||
_eager_temb_table_slices,
|
||||
_wan_temb_table_slices,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.platforms import current_platform
|
||||
|
||||
|
||||
class TestWanTembTableSlices(unittest.TestCase):
|
||||
@unittest.skipUnless(torch.cuda.is_available(), "requires CUDA")
|
||||
@unittest.skipUnless(
|
||||
torch.cuda.is_available() and current_platform.is_cuda(),
|
||||
"requires NVIDIA CUDA fusion",
|
||||
)
|
||||
def test_fused_matches_eager_and_is_contiguous(self):
|
||||
torch.manual_seed(0)
|
||||
for batch, seq, hidden in [(1, 517, 3072), (2, 64, 1536)]:
|
||||
|
||||
Reference in New Issue
Block a user