diff --git a/.github/workflows/pr-test-amd.yml b/.github/workflows/pr-test-amd.yml index 3c3e4fa962..a2a1f98127 100644 --- a/.github/workflows/pr-test-amd.yml +++ b/.github/workflows/pr-test-amd.yml @@ -866,14 +866,12 @@ jobs: echo "=== Post-test System Memory Status ===" free -h - # The mm_gen `unit` suite is portable CPU-style tests that finish in a - # few minutes, which does not justify a job of its own: on mi300 the - # container pull plus dependency install costs ~2h before any test runs. - # Ride along on the shard that is already warm. Runs after the diffusion - # tests, and on failure too, so a diffusion regression cannot hide it. + # Split unit files across the four existing containers to stay within + # the step timeout without repeating image pulls and dependency installs. + # Run after failed server tests too, so a regression cannot hide them. # Skips the CUDA-only ltx2_vae_channels_last memory-format assertion. - name: Run diffusion unit tests - if: matrix.part == 0 && (success() || failure()) + if: success() || failure() timeout-minutes: 30 run: | docker exec \ @@ -881,6 +879,8 @@ jobs: -w /sglang-checkout/python \ ci_sglang python3 sglang/multimodal_gen/test/run_suite.py \ --suite unit \ + --partition-id ${{ matrix.part }} \ + --total-partitions 4 \ -k "not ltx2_vae_channels_last" \ ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} diff --git a/python/sglang/kernels/jit/csrc/diffusion/gelu_tanh_cat.cuh b/python/sglang/kernels/jit/csrc/diffusion/gelu_tanh_cat.cuh index 9172bf0340..314a10ccb8 100644 --- a/python/sglang/kernels/jit/csrc/diffusion/gelu_tanh_cat.cuh +++ b/python/sglang/kernels/jit/csrc/diffusion/gelu_tanh_cat.cuh @@ -52,25 +52,33 @@ __global__ void gelu_tanh_cat_kernel( template void gelu_tanh_cat(tvm::ffi::TensorView attn, tvm::ffi::TensorView mlp, tvm::ffi::TensorView output) { using namespace host; - auto rows = SymbolicSize{"rows"}; - auto attn_width = SymbolicSize{"attn_width"}; - auto mlp_width = SymbolicSize{"mlp_width"}; + CHECK_HOST(attn.ndim() >= 2) << "gelu_tanh_cat: expected at least two dimensions"; + auto dtype = SymbolicDType{}; + dtype.set_options(); auto device_ = SymbolicDevice{}; device_.set_options(); - TensorMatcher({rows, attn_width}).with_dtype().with_device(device_).verify(attn); - TensorMatcher({rows, mlp_width}).with_dtype().with_device(device_).verify(mlp); - const int64_t a = attn_width.unwrap(); - const int64_t m = mlp_width.unwrap(); - TensorMatcher({rows, a + m}).with_dtype().with_device(device_).verify(output); + for (const auto tensor : {attn, mlp, output}) { + dtype.verify(tensor.dtype()); + device_.verify(tensor.device()); + CHECK_HOST(tensor.ndim() == attn.ndim() && tensor.is_contiguous()) + << "gelu_tanh_cat: expected contiguous tensors of equal rank"; + for (int64_t dim = 0; dim < attn.ndim() - 1; ++dim) { + CHECK_HOST(tensor.size(dim) == attn.size(dim)) << "gelu_tanh_cat: leading dimensions must match"; + } + } + const int64_t a = attn.size(-1); + const int64_t m = mlp.size(-1); + CHECK_HOST(output.size(-1) == a + m) << "gelu_tanh_cat: incorrect output width"; constexpr int kVecN = 16 / sizeof(T); CHECK_HOST(a > 0 && m > 0 && a % kVecN == 0 && m % kVecN == 0) << "gelu_tanh_cat: both widths must be positive multiples of " << kVecN; - CHECK_HOST(rows.unwrap() > 0) << "gelu_tanh_cat: rows must be positive"; + const int64_t rows = attn.numel() / a; + CHECK_HOST(rows > 0) << "gelu_tanh_cat: rows must be positive"; CHECK_HOST( reinterpret_cast(attn.data_ptr()) % 16 == 0 && reinterpret_cast(mlp.data_ptr()) % 16 == 0 && reinterpret_cast(output.data_ptr()) % 16 == 0) << "gelu_tanh_cat: tensors must be 16-byte aligned"; - const int64_t n = rows.unwrap() * ((a + m) / kVecN); + const int64_t n = rows * ((a + m) / kVecN); CHECK_HOST(n <= std::numeric_limits::max()) << "gelu_tanh_cat: tensor is too large"; constexpr int kBlockSize = 256; const auto kernel = gelu_tanh_cat_kernel; diff --git a/python/sglang/kernels/jit/csrc/diffusion/helios_qk_rope.cuh b/python/sglang/kernels/jit/csrc/diffusion/helios_qk_rope.cuh index ca601a7e4a..17a25c829a 100644 --- a/python/sglang/kernels/jit/csrc/diffusion/helios_qk_rope.cuh +++ b/python/sglang/kernels/jit/csrc/diffusion/helios_qk_rope.cuh @@ -19,7 +19,7 @@ namespace sglang { * multiply and add/subtract operations preserve the separate eager FP32 * intermediates before the result is rounded back to fp16/bf16. * - * \tparam T Activation type: fp16_t or bf16_t + * \tparam T Activation type: fp16_t, bf16_t or fp32_t * \param q Normalized query tensor, contiguous [tokens, heads, head_dim] * \param k Normalized key tensor, contiguous [tokens, heads, head_dim] * \param freqs Transposed Helios frequency tensor, contiguous @@ -38,7 +38,7 @@ __global__ void helios_qk_rope_kernel( uint32_t pairs_per_head, uint32_t num_heads, uint32_t freq_stride) { - static_assert(std::is_same_v || std::is_same_v); + static_assert(std::is_same_v || std::is_same_v || std::is_same_v); using Packed = packed_t; auto* q_pairs = reinterpret_cast(q); @@ -79,10 +79,18 @@ struct HeliosQKRoPEKernel { auto device = SymbolicDevice{}; device.set_options(); - TensorMatcher({N, H, D}).with_dtype().with_device(device).verify(q).verify(k); - TensorMatcher({N, F}).with_dtype().with_device(device).verify(freqs); + int64_t batch = 1; + if (q.ndim() == 4) { + auto B = SymbolicSize{"batch"}; + TensorMatcher({B, N, H, D}).with_dtype().with_device(device).verify(q).verify(k); + TensorMatcher({B, N, F}).with_dtype().with_device(device).verify(freqs); + batch = B.unwrap(); + } else { + TensorMatcher({N, H, D}).with_dtype().with_device(device).verify(q).verify(k); + TensorMatcher({N, F}).with_dtype().with_device(device).verify(freqs); + } - const int64_t tokens = N.unwrap(); + const int64_t tokens = batch * N.unwrap(); const int64_t heads = H.unwrap(); const int64_t head_dim = D.unwrap(); const int64_t freq_dim = F.unwrap(); diff --git a/python/sglang/kernels/jit/csrc/diffusion/interleaved_rope_fp64.cuh b/python/sglang/kernels/jit/csrc/diffusion/interleaved_rope_fp64.cuh index bcdd599b2f..7c56de7021 100644 --- a/python/sglang/kernels/jit/csrc/diffusion/interleaved_rope_fp64.cuh +++ b/python/sglang/kernels/jit/csrc/diffusion/interleaved_rope_fp64.cuh @@ -91,27 +91,27 @@ struct InterleavedRopeFP64Kernel { tvm::ffi::TensorView q, tvm::ffi::TensorView k, tvm::ffi::TensorView cos, - tvm::ffi::TensorView sin, - int64_t batch_size, - int64_t seq_len, - int64_t num_heads, - int64_t head_dim) { + tvm::ffi::TensorView sin) { using namespace host; using Pair = device::AlignedVector; - auto N = SymbolicSize{"activation_elements"}; - auto R = SymbolicSize{"table_elements"}; + auto B = SymbolicSize{"batch"}; + auto S = SymbolicSize{"sequence"}; + auto H = SymbolicSize{"heads"}; + auto D = SymbolicSize{"head_dim"}; auto device = SymbolicDevice{}; device.set_options(); - TensorMatcher({N}).with_dtype().with_device(device).verify(q_out).verify(k_out).verify(q).verify(k); - TensorMatcher({R}).with_dtype().with_device(device).verify(cos).verify(sin); + TensorMatcher({B, S, H, D}).with_dtype().with_device(device).verify(q_out).verify(k_out).verify(q).verify(k); + TensorMatcher({1, S, 1, D}).with_dtype().with_device(device).verify(cos).verify(sin); + + const int64_t batch_size = B.unwrap(); + const int64_t seq_len = S.unwrap(); + const int64_t num_heads = H.unwrap(); + const int64_t head_dim = D.unwrap(); CHECK_HOST(batch_size > 0 && seq_len > 0 && num_heads > 0 && head_dim > 0) << "interleaved_rope_fp64 dimensions must be positive"; CHECK_HOST(head_dim % 2 == 0) << "interleaved_rope_fp64 head_dim must be even"; - CHECK_HOST(N.unwrap() == batch_size * seq_len * num_heads * head_dim) - << "interleaved_rope_fp64 activation shape does not match dimensions"; - CHECK_HOST(R.unwrap() == seq_len * head_dim) << "interleaved_rope_fp64 table shape does not match dimensions"; CHECK_HOST( q_out.data_ptr() != k_out.data_ptr() && q_out.data_ptr() != q.data_ptr() && q_out.data_ptr() != k.data_ptr() && k_out.data_ptr() != q.data_ptr() && k_out.data_ptr() != k.data_ptr()) @@ -123,7 +123,7 @@ struct InterleavedRopeFP64Kernel { reinterpret_cast(k.data_ptr()) % alignof(Pair) == 0) << "interleaved_rope_fp64 activations must be aligned to rotation pairs"; - const int64_t num_pairs = N.unwrap() / 2; + const int64_t num_pairs = q.numel() / 2; const int64_t pairs_per_head = head_dim / 2; const auto blocks = static_cast(std::min(div_ceil(num_pairs, static_cast(kBlockSize)), kMaxGrid)); diff --git a/python/sglang/kernels/jit/csrc/diffusion/ltx25_decoder_rope.cuh b/python/sglang/kernels/jit/csrc/diffusion/ltx25_decoder_rope.cuh index 06e1662dbb..53be302f8e 100644 --- a/python/sglang/kernels/jit/csrc/diffusion/ltx25_decoder_rope.cuh +++ b/python/sglang/kernels/jit/csrc/diffusion/ltx25_decoder_rope.cuh @@ -122,27 +122,32 @@ struct LTX25DecoderRopeKernel { tvm::ffi::TensorView sin_h, tvm::ffi::TensorView cos_w, tvm::ffi::TensorView sin_w, - int64_t batch_size, - int64_t num_frames, - int64_t height, - int64_t width, - int64_t num_heads, - int64_t head_dim, int64_t dim_t, int64_t dim_h) { using namespace host; using Pair = device::AlignedVector; - auto N = SymbolicSize{"activation_elements"}; - auto RT = SymbolicSize{"temporal_table_elements"}; - auto RH = SymbolicSize{"height_table_elements"}; - auto RW = SymbolicSize{"width_table_elements"}; + auto B = SymbolicSize{"batch"}; + auto F = SymbolicSize{"frames"}; + auto H = SymbolicSize{"height"}; + auto W = SymbolicSize{"width"}; + auto A = SymbolicSize{"heads"}; + auto D = SymbolicSize{"head_dim"}; auto device = SymbolicDevice{}; device.set_options(); - TensorMatcher({N}).with_dtype().with_device(device).verify(q_out).verify(k_out).verify(q).verify(k); - TensorMatcher({RT}).with_dtype().with_device(device).verify(cos_t).verify(sin_t); - TensorMatcher({RH}).with_dtype().with_device(device).verify(cos_h).verify(sin_h); - TensorMatcher({RW}).with_dtype().with_device(device).verify(cos_w).verify(sin_w); + TensorMatcher({B, F, H, W, A, D}) + .with_dtype() + .with_device(device) + .verify(q_out) + .verify(k_out) + .verify(q) + .verify(k); + const int64_t batch_size = B.unwrap(); + const int64_t num_frames = F.unwrap(); + const int64_t height = H.unwrap(); + const int64_t width = W.unwrap(); + const int64_t num_heads = A.unwrap(); + const int64_t head_dim = D.unwrap(); CHECK_HOST(batch_size > 0 && num_frames > 0 && height > 0 && width > 0 && num_heads > 0 && head_dim > 0) << "ltx25_decoder_rope dimensions must be positive"; @@ -150,11 +155,9 @@ struct LTX25DecoderRopeKernel { << "ltx25_decoder_rope dimensions must split into positive rotation pairs"; const int64_t dim_w = head_dim - dim_t - dim_h; CHECK_HOST(dim_w > 0 && dim_w % 2 == 0) << "ltx25_decoder_rope width dimension must be positive and even"; - CHECK_HOST(N.unwrap() == batch_size * num_frames * height * width * num_heads * head_dim) - << "ltx25_decoder_rope activation shape does not match dimensions"; - CHECK_HOST(RT.unwrap() == num_frames * dim_t / 2) << "ltx25_decoder_rope temporal table shape mismatch"; - CHECK_HOST(RH.unwrap() == height * dim_h / 2) << "ltx25_decoder_rope height table shape mismatch"; - CHECK_HOST(RW.unwrap() == width * dim_w / 2) << "ltx25_decoder_rope width table shape mismatch"; + TensorMatcher({F, dim_t / 2}).with_dtype().with_device(device).verify(cos_t).verify(sin_t); + TensorMatcher({H, dim_h / 2}).with_dtype().with_device(device).verify(cos_h).verify(sin_h); + TensorMatcher({W, dim_w / 2}).with_dtype().with_device(device).verify(cos_w).verify(sin_w); CHECK_HOST( q_out.data_ptr() != k_out.data_ptr() && q_out.data_ptr() != q.data_ptr() && q_out.data_ptr() != k.data_ptr() && k_out.data_ptr() != q.data_ptr() && k_out.data_ptr() != k.data_ptr()) @@ -166,7 +169,7 @@ struct LTX25DecoderRopeKernel { reinterpret_cast(k.data_ptr()) % alignof(Pair) == 0) << "ltx25_decoder_rope activations must be aligned to rotation pairs"; - const int64_t num_pairs = N.unwrap() / 2; + const int64_t num_pairs = q.numel() / 2; const int64_t pairs_per_head = head_dim / 2; const auto blocks = static_cast(std::min(div_ceil(num_pairs, static_cast(kBlockSize)), kMaxGrid)); diff --git a/python/sglang/kernels/jit/csrc/diffusion/qknorm_complex_rope.cuh b/python/sglang/kernels/jit/csrc/diffusion/qknorm_complex_rope.cuh index bd8f0cee0f..b4ed963cc6 100644 --- a/python/sglang/kernels/jit/csrc/diffusion/qknorm_complex_rope.cuh +++ b/python/sglang/kernels/jit/csrc/diffusion/qknorm_complex_rope.cuh @@ -211,7 +211,8 @@ __global__ void kernel(const Params __grid_constant__ params) { template struct Kernel { - /// \brief Single tensor, out of place: `out = rope(rmsnorm(x))` for a contiguous [B, S, H, 128] bf16 tensor. + /// \brief Single tensor, out of place: `out = rope(rmsnorm(x))` for a [B, S, H, 128] bf16 tensor with packed head + /// rows. static void run(tvm::ffi::TensorView x, tvm::ffi::TensorView out, @@ -224,19 +225,31 @@ struct Kernel { auto H = SymbolicSize{"heads"}; auto device = SymbolicDevice{}; device.set_options(); + auto Bx = SymbolicSize{"input_batch_stride"}; + auto Tx = SymbolicSize{"input_token_stride"}; + TensorMatcher({B, S, H, kHeadDim}) + .with_strides({Bx, Tx, kHeadDim, 1}) + .with_dtype() + .with_device(device) + .ensure_alignment(kVecAlignBytes) + .verify(x); TensorMatcher({B, S, H, kHeadDim}) .with_dtype() .with_device(device) .ensure_alignment(kVecAlignBytes) - .verify(x) .verify(out); TensorMatcher({kHeadDim}).with_dtype().with_device(device).verify(weight); - TensorMatcher({S, kHeadDim}).with_dtype().with_device(device).ensure_alignment(16).verify(rope); + TensorMatcher({S, kHeadDim / 2, 2}).with_dtype().with_device(device).verify(rope); + // float4 loads span two complex pairs; only the token rows need 16-byte alignment. + CHECK_HOST(reinterpret_cast(rope.data_ptr()) % 16 == 0) << "RoPE cache must be 16-byte aligned"; + CHECK_HOST( + Bx.unwrap() % (kVecAlignBytes / sizeof(bf16_t)) == 0 && Tx.unwrap() % (kVecAlignBytes / sizeof(bf16_t)) == 0) + << "input batch/token strides must preserve vector alignment"; const int64_t batch = B.unwrap(), seq = S.unwrap(), heads = H.unwrap(); if (batch == 0 || seq == 0 || heads == 0) return; const int64_t token_stride = heads * kHeadDim; Params params{}; - params.q_src = Rows{static_cast(x.data_ptr()), seq * token_stride, token_stride}; + params.q_src = Rows{static_cast(x.data_ptr()), Bx.unwrap(), Tx.unwrap()}; params.q_dst = MutableRows{static_cast(out.data_ptr()), seq * token_stride, token_stride}; params.q_weight = static_cast(weight.data_ptr()); params.k_weight = params.q_weight; @@ -301,7 +314,9 @@ struct Kernel { .verify(k_prefix) .verify(v_prefix); TensorMatcher({kHeadDim}).with_dtype().with_device(device).verify(q_weight).verify(k_weight); - TensorMatcher({S, kHeadDim}).with_dtype().with_device(device).ensure_alignment(16).verify(rope); + TensorMatcher({S, kHeadDim / 2, 2}).with_dtype().with_device(device).verify(rope); + // float4 loads span two complex pairs; only the token rows need 16-byte alignment. + CHECK_HOST(reinterpret_cast(rope.data_ptr()) % 16 == 0) << "RoPE cache must be 16-byte aligned"; const int64_t batch = B.unwrap(), seq = S.unwrap(), heads = H.unwrap(), prefix = P.unwrap(); CHECK_HOST(PS.unwrap() == prefix + seq) << "k_out/v_out must hold prefix + seq tokens"; if (batch == 0 || seq == 0 || heads == 0) return; diff --git a/python/sglang/kernels/jit/csrc/diffusion/vdn_delta_factors.cuh b/python/sglang/kernels/jit/csrc/diffusion/vdn_delta_factors.cuh index 1a91cb4bd1..110c3bee25 100644 --- a/python/sglang/kernels/jit/csrc/diffusion/vdn_delta_factors.cuh +++ b/python/sglang/kernels/jit/csrc/diffusion/vdn_delta_factors.cuh @@ -185,11 +185,11 @@ SGL_DEVICE void step(float2 (&t)[8][4], Smem& sm, int kt, int ti, int tj, int i0 /** * \brief transition = diag(alpha) (I + A)^-1, injection = B (I + A)^-1 for a batch of 128x128 SPD A. * - * \param A [N, 128, 128] fp32, symmetric positive semi-definite (I + A is inverted) - * \param B [N, 128, 128] fp32 - * \param alpha [N, 128] fp32 row scales of the transition - * \param transition [N, 128, 128] fp32 output - * \param injection [N, 128, 128] fp32 output + * \param A [..., 128, 128] fp32, symmetric positive semi-definite (I + A is inverted) + * \param B [..., 128, 128] fp32 + * \param alpha [..., 128] fp32 row scales of the transition + * \param transition [..., 128, 128] fp32 output + * \param injection [..., 128, 128] fp32 output */ __global__ void __launch_bounds__(kBlockSize, kMinBlocksPerSm) vdn_delta_factors_kernel( const float* __restrict__ A, @@ -305,11 +305,11 @@ struct VdnDeltaFactorsKernel { /** * \brief Validate the tensors and launch one CTA per matrix. * - * \param transition [N, 128, 128] fp32 output, diag(alpha) (I + A)^-1 - * \param injection [N, 128, 128] fp32 output, B (I + A)^-1 - * \param A [N, 128, 128] fp32 SPD statistics (I + A is inverted) - * \param B [N, 128, 128] fp32 - * \param alpha [N, 128] fp32 + * \param transition [..., 128, 128] fp32 output, diag(alpha) (I + A)^-1 + * \param injection [..., 128, 128] fp32 output, B (I + A)^-1 + * \param A [..., 128, 128] fp32 SPD statistics (I + A is inverted) + * \param B [..., 128, 128] fp32 + * \param alpha [..., 128] fp32 */ static void run(tvm::ffi::TensorView transition, @@ -318,18 +318,28 @@ struct VdnDeltaFactorsKernel { tvm::ffi::TensorView B, tvm::ffi::TensorView alpha) { using namespace host; - auto N = SymbolicSize{"num_matrices"}; + CHECK_HOST(A.ndim() >= 2 && A.size(-2) == kDim && A.size(-1) == kDim) + << "vdn_delta_factors expects [..., 128, 128] matrices"; + auto dtype = SymbolicDType{}; + dtype.set_options(); auto device = SymbolicDevice{}; device.set_options(); - TensorMatcher({N, kDim, kDim}) - .with_dtype() - .with_device(device) - .verify(transition) - .verify(injection) - .verify(A) - .verify(B); - TensorMatcher({N, kDim}).with_dtype().with_device(device).verify(alpha); - const int64_t num = N.unwrap(); + for (const auto tensor : {A, B, transition, injection, alpha}) { + dtype.verify(tensor.dtype()); + device.verify(tensor.device()); + CHECK_HOST(tensor.is_contiguous()) << "vdn_delta_factors expects contiguous tensors"; + } + for (const auto tensor : {B, transition, injection}) { + CHECK_HOST(tensor.ndim() == A.ndim()) << "vdn_delta_factors matrix ranks must match"; + for (int64_t dim = 0; dim < A.ndim(); ++dim) { + CHECK_HOST(tensor.size(dim) == A.size(dim)) << "vdn_delta_factors matrix shapes must match"; + } + } + CHECK_HOST(alpha.ndim() == A.ndim() - 1) << "alpha must have shape A.shape[:-1]"; + for (int64_t dim = 0; dim < alpha.ndim(); ++dim) { + CHECK_HOST(alpha.size(dim) == A.size(dim)) << "alpha must have shape A.shape[:-1]"; + } + const int64_t num = A.numel() / (kDim * kDim); if (num == 0) return; CHECK_HOST( transition.data_ptr() != A.data_ptr() && transition.data_ptr() != B.data_ptr() && diff --git a/python/sglang/kernels/kda_kernels/causal_conv3d_cat_pad_jit.py b/python/sglang/kernels/kda_kernels/causal_conv3d_cat_pad_jit.py index 896873a4d1..3b12f38f3b 100644 --- a/python/sglang/kernels/kda_kernels/causal_conv3d_cat_pad_jit.py +++ b/python/sglang/kernels/kda_kernels/causal_conv3d_cat_pad_jit.py @@ -17,6 +17,8 @@ _SUPPORTED_DTYPES = (torch.float16, torch.bfloat16, torch.float32) @cache_once def _jit_causal_conv3d_cat_pad_module(dtype: torch.dtype) -> Module: + if dtype not in _SUPPORTED_DTYPES: + raise RuntimeError(f"unsupported dtype for causal Conv3D cat/pad: {dtype}") args = make_cpp_args(dtype) return load_jit( "diffusion_causal_conv3d_cat_pad", @@ -101,19 +103,7 @@ def fused_causal_conv3d_cat_pad_cuda( cache_x: torch.Tensor, padding: list[int] | tuple[int, ...], ) -> torch.Tensor: - if x.dtype not in _SUPPORTED_DTYPES: - raise RuntimeError(f"unsupported dtype for causal Conv3D cat/pad: {x.dtype}") - if not torch.compiler.is_compiling(): - if ( - not x.is_cuda - or not cache_x.is_cuda - or x.dim() != 5 - or cache_x.dim() != 5 - or not x.is_contiguous() - or not cache_x.is_contiguous() - or not can_use_fused_causal_conv3d_cat_pad_cuda(x, cache_x, padding) - ): - raise RuntimeError("unsupported input for causal Conv3D cat/pad CUDA") + """Concatenate the temporal cache and pad; the C++ launcher validates inputs.""" return _causal_conv3d_cat_pad_custom_op(x, cache_x, *padding) diff --git a/python/sglang/kernels/kda_kernels/csrc/diffusion/residual_gate_add.cuh b/python/sglang/kernels/kda_kernels/csrc/diffusion/residual_gate_add.cuh index bd74391ef8..ab865ed6d6 100644 --- a/python/sglang/kernels/kda_kernels/csrc/diffusion/residual_gate_add.cuh +++ b/python/sglang/kernels/kda_kernels/csrc/diffusion/residual_gate_add.cuh @@ -26,8 +26,6 @@ constexpr uint32_t kBroadcastRowsPerBlock = 4; constexpr uint32_t kBroadcastColsPerBlock = 256; constexpr uint32_t kMaxGrid = 65535; constexpr uintptr_t kAlignment = 16; -constexpr uint32_t kTransposeTile = 32; -constexpr uint32_t kTransposeBlockSize = 256; enum class GateMode : int { kFull, kBroadcastRow, kPerToken }; @@ -116,55 +114,6 @@ __global__ void residual_gate_add_scalar_kernel( } } -/** - * Fuse the residual-gate update when the residual/output use the common - * ``[batch, hidden, tokens]`` backing layout exposed as ``[batch, tokens, - * hidden]``, while update remains contiguous in the logical layout. - * - * Loading update in logical row-major order and consuming it transposed from - * shared memory keeps both update reads and residual/output traffic coalesced. - */ -template -__global__ void residual_gate_add_transposed_kernel( - T* __restrict__ out, - const T* __restrict__ residual, - const T* __restrict__ update, - const T* __restrict__ gate, - int64_t tokens, - int64_t hidden_size) { - __shared__ T update_tile[kTransposeTile][kTransposeTile + 1]; - - const int64_t batch = blockIdx.z; - const int64_t token_base = static_cast(blockIdx.x) * kTransposeTile; - const int64_t hidden_base = static_cast(blockIdx.y) * kTransposeTile; - const int64_t batch_offset = batch * tokens * hidden_size; - -#pragma unroll - for (uint32_t item = threadIdx.x; item < kTransposeTile * kTransposeTile; item += kTransposeBlockSize) { - const uint32_t token_in_tile = item / kTransposeTile; - const uint32_t hidden_in_tile = item % kTransposeTile; - const int64_t token = token_base + token_in_tile; - const int64_t hidden = hidden_base + hidden_in_tile; - if (token < tokens && hidden < hidden_size) { - update_tile[token_in_tile][hidden_in_tile] = update[batch_offset + token * hidden_size + hidden]; - } - } - __syncthreads(); - -#pragma unroll - for (uint32_t item = threadIdx.x; item < kTransposeTile * kTransposeTile; item += kTransposeBlockSize) { - const uint32_t hidden_in_tile = item / kTransposeTile; - const uint32_t token_in_tile = item % kTransposeTile; - const int64_t token = token_base + token_in_tile; - const int64_t hidden = hidden_base + hidden_in_tile; - if (token < tokens && hidden < hidden_size) { - const int64_t transposed_offset = batch_offset + hidden * tokens + token; - out[transposed_offset] = residual_gate_value( - residual[transposed_offset], update_tile[token_in_tile][hidden_in_tile], SGLANG_LDG(gate + hidden)); - } - } -} - } // namespace /** @@ -281,49 +230,6 @@ struct ResidualGateAddKernel { hidden_size); } } - - static void run_transposed( - tvm::ffi::TensorView out, tvm::ffi::TensorView residual, tvm::ffi::TensorView update, tvm::ffi::TensorView gate) { - using namespace host; - - auto B = SymbolicSize{"batch"}; - auto S = SymbolicSize{"tokens"}; - auto D = SymbolicSize{"hidden_size"}; - auto device = SymbolicDevice{}; - device.set_options(); - TensorMatcher({B, S, D}).with_strides({-1, 1, -1}).with_dtype().with_device(device).verify(out).verify(residual); - TensorMatcher({B, S, D}).with_strides({-1, -1, 1}).with_dtype().with_device(device).verify(update); - TensorMatcher({1, 1, D}).with_strides({-1, -1, 1}).with_dtype().with_device(device).verify(gate); - - const int64_t batch = B.unwrap(); - const int64_t tokens = S.unwrap(); - const int64_t hidden_size = D.unwrap(); - CHECK_HOST(batch > 0 && tokens > 0 && hidden_size > 0) << "transposed residual-gate tensors must be non-empty"; - CHECK_HOST( - batch <= kMaxGrid && div_ceil(tokens, int64_t{kTransposeTile}) <= kMaxGrid && - div_ceil(hidden_size, int64_t{kTransposeTile}) <= kMaxGrid) - << "transposed residual-gate grid exceeds CUDA limits"; - const int64_t batch_stride = tokens * hidden_size; - CHECK_HOST( - out.stride(0) == batch_stride && out.stride(2) == tokens && residual.stride(0) == batch_stride && - residual.stride(2) == tokens) - << "residual/output must use the transposed dense layout"; - CHECK_HOST(update.stride(0) == batch_stride && update.stride(1) == hidden_size) << "update must be contiguous"; - - auto* out_ptr = static_cast(out.data_ptr()); - const auto* residual_ptr = static_cast(residual.data_ptr()); - const auto* update_ptr = static_cast(update.data_ptr()); - const auto* gate_ptr = static_cast(gate.data_ptr()); - CHECK_HOST(out_ptr != residual_ptr && out_ptr != update_ptr && out_ptr != gate_ptr) - << "output must not alias an input"; - - const dim3 grid( - static_cast(div_ceil(tokens, int64_t{kTransposeTile})), - static_cast(div_ceil(hidden_size, int64_t{kTransposeTile})), - static_cast(batch)); - LaunchKernel(grid, kTransposeBlockSize, device.unwrap())( - residual_gate_add_transposed_kernel, out_ptr, residual_ptr, update_ptr, gate_ptr, tokens, hidden_size); - } }; } // namespace residual_gate_add diff --git a/python/sglang/kernels/kda_kernels/layernorm_modulate_triton.py b/python/sglang/kernels/kda_kernels/layernorm_modulate_triton.py index ec4c1e32b6..db9c1ca028 100644 --- a/python/sglang/kernels/kda_kernels/layernorm_modulate_triton.py +++ b/python/sglang/kernels/kda_kernels/layernorm_modulate_triton.py @@ -402,33 +402,45 @@ def _qk_head_launch_config() -> tuple[int, int]: def _mod_row_stride(t: torch.Tensor, batch: int, hidden: int) -> int | None: # (batch, hidden) modulation rows, possibly strided views of a chunked # adaLN projection; the last dim must be packed. - if t.dim() != 2 or t.shape != (batch, hidden) or t.stride(1) != 1: + if t.shape != (batch, hidden) or t.stride(1) != 1: return None return t.stride(0) if batch > 1 else hidden -def can_use_fused_layernorm_modulate( - x: torch.Tensor, scale: torch.Tensor, shift: torch.Tensor | None -) -> bool: +def can_use_fused_layernorm_modulate(dtype: torch.dtype, hidden: int) -> bool: + """Select the BF16 vectorized LayerNorm reduction replicated below.""" + return ( + is_cuda() and dtype is torch.bfloat16 and 0 < hidden <= 8192 and hidden % 4 == 0 + ) + + +def _validate_layernorm_modulate(x, scale, shift) -> tuple[int, int, int, int]: + shape = x.shape if not ( _is_bf16_cuda(x) - and x.dim() == 3 + and len(shape) == 3 and x.numel() > 0 and x.is_contiguous() - and x.shape[-1] % 4 == 0 - and x.shape[-1] <= 8192 - and _is_bf16_cuda(scale) - and scale.device == x.device + and 0 < shape[-1] <= 8192 + and shape[-1] % 4 == 0 ): - return False - batch, _, hidden = x.shape - q = _mod_row_stride(scale, batch, hidden) - if shift is None: - return q is not None - if not _is_bf16_cuda(shift) or shift.device != x.device: - return False - v = _mod_row_stride(shift, batch, hidden) - return q is not None and v is not None and q == v + raise RuntimeError( + "LayerNorm modulation expects contiguous BF16 CUDA [B, S, D], D divisible by 4 and <= 8192" + ) + batch, seq_len, hidden = shape + device = x.device + stride = _mod_row_stride(scale, batch, hidden) + if not ( + stride is not None and scale.dtype is torch.bfloat16 and scale.device == device + ): + raise RuntimeError("scale must have packed BF16 [B, D] rows on x's device") + if shift is not None and not ( + shift.dtype is torch.bfloat16 + and shift.device == device + and _mod_row_stride(shift, batch, hidden) == stride + ): + raise RuntimeError("shift must match scale's row layout and dtype/device") + return batch, seq_len, hidden, stride def _fake_ln_modulate( @@ -510,11 +522,10 @@ def fused_layernorm_modulate_raw( of microseconds per call); use it on CPU-launch-bound eager hot paths (e.g. Sana), and the registered custom op under ``torch.compile``. """ - batch, seq_len, hidden = x.shape + batch, seq_len, hidden, stride = _validate_layernorm_modulate(x, scale, shift) n_rows = batch * seq_len rows = 2 out = torch.empty_like(x) - stride = _mod_row_stride(scale, batch, hidden) with torch.cuda.device(x.device): _layernorm_modulate_kernel[(triton.cdiv(n_rows, rows),)]( out, @@ -558,11 +569,10 @@ def fused_layernorm_modulate_fp8_quant_raw( Unlike that two-op chain, this path does not materialize the intermediate BF16 activation because FLUX.2 feeds it directly into an FP8 projection. """ - batch, seq_len, hidden = x.shape + batch, seq_len, hidden, stride = _validate_layernorm_modulate(x, scale, shift) n_rows = batch * seq_len rows = 2 out = torch.empty_like(x, dtype=fp8_dtype) - stride = _mod_row_stride(scale, batch, hidden) with torch.cuda.device(x.device): _layernorm_modulate_kernel[(triton.cdiv(n_rows, rows),)]( out, @@ -595,19 +605,13 @@ fused_layernorm_modulate = register_custom_op( ) -def can_use_fused_qk_head_layernorm(q: torch.Tensor, k: torch.Tensor) -> bool: - head_dim = q.shape[-1] if q.dim() == 4 else 0 +def can_use_fused_qk_head_layernorm(dtype: torch.dtype, head_dim: int) -> bool: + """Select the BF16 per-head LayerNorm reduction.""" return ( - _is_bf16_cuda(q) - and _is_bf16_cuda(k) - and q.device == k.device - and q.dim() == 4 - and q.shape == k.shape - and head_dim % 4 == 0 + is_cuda() + and dtype is torch.bfloat16 and 0 < head_dim <= 128 - and q.numel() > 0 - and q.is_contiguous() - and k.is_contiguous() + and head_dim % 4 == 0 ) @@ -627,7 +631,23 @@ def fused_qk_head_layernorm( ) -> tuple[torch.Tensor, torch.Tensor]: """Per-head ``nn.LayerNorm(dim_head)`` (no affine) over q and k in one launch, bit-exact vs the eager aten kernel.""" - head_dim = q.shape[-1] + shape = q.shape + if not ( + _is_bf16_cuda(q) + and len(shape) == 4 + and q.numel() > 0 + and 0 < shape[-1] <= 128 + and shape[-1] % 4 == 0 + and k.dtype is torch.bfloat16 + and k.device == q.device + and k.shape == shape + and q.is_contiguous() + and k.is_contiguous() + ): + raise RuntimeError( + "QK LayerNorm expects matching contiguous BF16 CUDA [B, S, H, D], D divisible by 4 and <= 128" + ) + head_dim = shape[-1] n_rows = q.numel() // head_dim # Architecture sweeps at the production GLM shape select 32 rows / 1 warp # on B300 (SM103) and 8 rows / 4 warps on RTX 5090 (SM120). Preserve the diff --git a/python/sglang/kernels/kda_kernels/residual_gate_add_jit.py b/python/sglang/kernels/kda_kernels/residual_gate_add_jit.py index c11b113e7a..53e142da6e 100644 --- a/python/sglang/kernels/kda_kernels/residual_gate_add_jit.py +++ b/python/sglang/kernels/kda_kernels/residual_gate_add_jit.py @@ -1,6 +1,5 @@ from __future__ import annotations -import logging from typing import TYPE_CHECKING import torch @@ -19,9 +18,6 @@ _SUPPORTED_DTYPES = (torch.float16, torch.bfloat16, torch.float32) _BIT_EXACT_DTYPES = (torch.float16, torch.bfloat16) _TRANSPOSE_TILE = 32 _MAX_GRID_DIM = 65535 -_FAILED_RUNTIME_KEYS: set[tuple[int | None, torch.dtype]] = set() - -logger = logging.getLogger(__name__) @cache_once @@ -38,10 +34,6 @@ def _jit_residual_gate_add_module(dtype: torch.dtype) -> Module: "residual_gate_add", f"residual_gate_add::ResidualGateAddKernel<{args}>::run", ), - ( - "residual_gate_add_transposed", - f"residual_gate_add::ResidualGateAddKernel<{args}>::run_transposed", - ), ], ) @@ -67,9 +59,6 @@ def _residual_gate_add_cuda_impl( device=residual.device, ) module = _jit_residual_gate_add_module(residual.dtype) - if _is_transposed_dense_residual(residual, update, gate): - module.residual_gate_add_transposed(out, residual, update, gate) - return out gate_mode = _gate_mode(residual, gate) module.residual_gate_add( out.view(-1), @@ -96,120 +85,6 @@ def _round16_f32(x, IS_BF16: tl.constexpr): return bits.to(tl.float16, bitcast=True).to(tl.float32) -@triton.jit -def _store16(out_ptr, offs, value_f32, mask, IS_BF16: tl.constexpr): - if IS_BF16: - bits = tl.inline_asm_elementwise( - "cvt.rn.bf16.f32 $0, $1;", - "=h,r", - [value_f32], - dtype=tl.int16, - is_pure=True, - pack=1, - ) - tl.store(out_ptr + offs, bits.to(tl.bfloat16, bitcast=True), mask=mask) - else: - bits = tl.inline_asm_elementwise( - "cvt.rn.f16.f32 $0, $1;", - "=h,r", - [value_f32], - dtype=tl.int16, - is_pure=True, - pack=1, - ) - tl.store(out_ptr + offs, bits.to(tl.float16, bitcast=True), mask=mask) - - -@triton.jit -def _rga_flat( - out, - res, - upd, - gate, - numel, - hid, - MODE: tl.constexpr, - IS_BF16: tl.constexpr, - IS_16: tl.constexpr, - BLK: tl.constexpr, -): - pid = tl.program_id(0).to(tl.int64) - offs = pid * BLK + tl.arange(0, BLK).to(tl.int64) - mask = offs < numel - rv = tl.load(res + offs, mask=mask) - uv = tl.load(upd + offs, mask=mask) - if MODE == 1: - gv = tl.load(gate + (offs % hid), mask=mask) - elif MODE == 2: - gv = tl.load(gate + (offs // hid), mask=mask) - else: - gv = tl.load(gate + offs, mask=mask) - if IS_16: - p32 = uv.to(tl.float32) * gv.to(tl.float32) - pf = _round16_f32(p32, IS_BF16) - o32 = rv.to(tl.float32) + pf - _store16(out, offs, o32, mask, IS_BF16) - else: - p32 = uv * gv - o32 = rv + p32 - tl.store(out + offs, o32, mask=mask) - - -@triton.jit -def _rga_row( - out, - res, - upd, - gate, - hid: tl.constexpr, - IS_BF16: tl.constexpr, - EVG: tl.constexpr, - CG: tl.constexpr, - BLK: tl.constexpr, -): - row = tl.program_id(0).to(tl.int64) - offs = tl.arange(0, BLK) - mask = offs < hid - if EVG: - gv = tl.load(gate + offs, mask=mask, eviction_policy="evict_last") - else: - gv = tl.load(gate + offs, mask=mask) - base = row * hid - if CG: - rv = tl.load(res + base + offs, mask=mask, cache_modifier=".cg") - uv = tl.load(upd + base + offs, mask=mask, cache_modifier=".cg") - else: - rv = tl.load(res + base + offs, mask=mask, eviction_policy="evict_first") - uv = tl.load(upd + base + offs, mask=mask, eviction_policy="evict_first") - p32 = uv.to(tl.float32) * gv.to(tl.float32) - pf = _round16_f32(p32, IS_BF16) - o32 = rv.to(tl.float32) + pf - _store16(out, base + offs, o32, mask, IS_BF16) - - -@triton.jit -def _rga_flat_m1( - out, - res, - upd, - gate, - numel, - hid: tl.constexpr, - IS_BF16: tl.constexpr, - BLK: tl.constexpr, -): - pid = tl.program_id(0).to(tl.int64) - offs = pid * BLK + tl.arange(0, BLK).to(tl.int64) - mask = offs < numel - rv = tl.load(res + offs, mask=mask, eviction_policy="evict_first") - uv = tl.load(upd + offs, mask=mask, eviction_policy="evict_first") - gv = tl.load(gate + (offs % hid), mask=mask, eviction_policy="evict_last") - p32 = uv.to(tl.float32) * gv.to(tl.float32) - pf = _round16_f32(p32, IS_BF16) - o32 = rv.to(tl.float32) + pf - _store16(out, offs, o32, mask, IS_BF16) - - @triton.jit def _rga_transposed( out, @@ -268,146 +143,30 @@ def _rga_transposed( tl.store(out + base + roffs, rv + uv * gv[:, None], mask=m2) -def _residual_gate_add_triton(residual, update, gate): - # Both callers validate the inputs first; repeating the predicate here cost - # ~2.7us of host time on every call. +def _residual_gate_add_transposed(residual, update, gate): out = torch.empty_strided( residual.shape, residual.stride(), dtype=residual.dtype, device=residual.device ) - is_bf16 = residual.dtype == torch.bfloat16 - dtype16 = residual.dtype in (torch.float16, torch.bfloat16) - - if _is_transposed_dense_residual(residual, update, gate): - tokens = residual.shape[1] - hid = residual.shape[2] - batch = residual.shape[0] - if tokens * hid * batch <= 65536: - tile, warps = 16, 4 - elif tokens * hid * batch >= 1 << 20: - tile, warps = 64, 8 - else: - tile, warps = 32, 8 - grid = (triton.cdiv(tokens, tile), triton.cdiv(hid, tile), batch) - _rga_transposed[grid]( - out, - residual, - update, - gate, - tokens, - hid, - is_bf16, - dtype16, - TILE=tile, - num_warps=warps, - ) - return out - - if not dtype16: - # fp32: exact fp32 math, plain Triton elementwise - numel = residual.numel() - hid = residual.shape[-1] - if gate.shape == residual.shape: - mode = 0 - elif _is_row_broadcast_gate(residual, gate): - mode = 1 - else: - mode = 2 - if numel <= 8192: - _rga_flat[(triton.cdiv(numel, 256),)]( - out, - residual, - update, - gate, - numel, - hid, - mode, - False, - False, - BLK=256, - num_warps=4, - ) - else: - _rga_flat[(triton.cdiv(numel, 1024),)]( - out, - residual, - update, - gate, - numel, - hid, - mode, - False, - False, - BLK=1024, - num_warps=4, - ) - return out - - hid = residual.shape[-1] - numel = residual.numel() - if _is_row_broadcast_gate(residual, gate): - rows = numel // hid - if hid >= 4096 or (rows <= 512 and numel >= 1 << 20): - _rga_flat_m1[(triton.cdiv(numel, 1024),)]( - out, residual, update, gate, numel, hid, is_bf16, BLK=1024, num_warps=4 - ) - elif hid <= 16384: - blk = triton.next_power_of_2(hid) - _rga_row[(rows,)]( - out, - residual, - update, - gate, - hid, - is_bf16, - hid < 4608, - hid >= 4096, - BLK=blk, - num_warps=4 if hid >= 4608 else 8, - ) - else: - _rga_flat[(triton.cdiv(numel, 1024),)]( - out, - residual, - update, - gate, - numel, - hid, - 1, - is_bf16, - True, - BLK=1024, - num_warps=4, - ) - return out - mode = 0 if gate.shape == residual.shape else 2 - if numel <= 8192: - _rga_flat[(triton.cdiv(numel, 256),)]( - out, - residual, - update, - gate, - numel, - hid, - mode, - is_bf16, - True, - BLK=256, - num_warps=4, - ) + batch, tokens, hidden = residual.shape + if residual.numel() <= 65536: + tile, warps = 16, 4 + elif residual.numel() >= 1 << 20: + tile, warps = 64, 8 else: - _rga_flat[(triton.cdiv(numel, 1024),)]( - out, - residual, - update, - gate, - numel, - hid, - mode, - is_bf16, - True, - BLK=1024, - num_warps=4, - ) + tile, warps = 32, 8 + grid = (triton.cdiv(tokens, tile), triton.cdiv(hidden, tile), batch) + _rga_transposed[grid]( + out, + residual, + update, + gate, + tokens, + hidden, + residual.dtype == torch.bfloat16, + residual.dtype in (torch.float16, torch.bfloat16), + TILE=tile, + num_warps=warps, + ) return out @@ -424,7 +183,7 @@ def _residual_gate_add_custom_op( # benchmarked on. For the contiguous layouts every other diffusion model # uses, the existing JIT CUDA kernel is faster at every measured shape. if _is_transposed_dense_residual(residual, update, gate): - return _residual_gate_add_triton(residual, update, gate) + return _residual_gate_add_transposed(residual, update, gate) return _residual_gate_add_cuda_impl(residual, update, gate) @@ -472,7 +231,8 @@ def can_use_residual_gate_add_cuda( residual: torch.Tensor, update: torch.Tensor, gate: torch.Tensor ) -> bool: return ( - residual.dtype in _SUPPORTED_DTYPES + torch.version.hip is None + and residual.dtype in _SUPPORTED_DTYPES and residual.dtype == update.dtype and residual.dtype == gate.dtype and residual.is_cuda @@ -506,29 +266,11 @@ def residual_gate_add_cuda( def residual_gate_add( residual: torch.Tensor, update: torch.Tensor, gate: torch.Tensor ) -> torch.Tensor: - """Use the bit-exact CUDA fast path when supported, otherwise eager. - - Runtime build failures are cached per device and dtype so every diffusion - model shares one fallback policy instead of maintaining model-local flags. - """ - runtime_key = (residual.device.index, residual.dtype) - if ( - residual.dtype in _BIT_EXACT_DTYPES - and runtime_key not in _FAILED_RUNTIME_KEYS - and can_use_residual_gate_add_cuda(residual, update, gate) + """Use the bit-exact fast path for supported layouts, otherwise eager.""" + if residual.dtype in _BIT_EXACT_DTYPES and can_use_residual_gate_add_cuda( + residual, update, gate ): - try: - return _residual_gate_add_custom_op(residual, update, gate) - except Exception as exc: - if torch.compiler.is_compiling(): - raise - _FAILED_RUNTIME_KEYS.add(runtime_key) - logger.warning( - "Disabling diffusion residual-gate CUDA fast path on %s/%s: %s", - residual.device, - residual.dtype, - exc, - ) + return _residual_gate_add_custom_op(residual, update, gate) return residual + update * gate diff --git a/python/sglang/kernels/ops/diffusion/README.md b/python/sglang/kernels/ops/diffusion/README.md index 29d38e25c8..24722c37d8 100644 --- a/python/sglang/kernels/ops/diffusion/README.md +++ b/python/sglang/kernels/ops/diffusion/README.md @@ -80,18 +80,41 @@ GEMM in FP32. The default path still promotes Q/K/V before both GEMMs. ## Entry-point protocol -Every public kernel is a **predicate + kernel** pair: +Keep backend selection separate from input validation. -```python -if can_use_(...): - out = (...) -else: - out = -``` +- Select a backend using properties that distinguish supported implementations: + device family, dtype, layout, alignment, architecture, and numerical contract. + A shape restriction belongs here only when it is a real specialization, such + as a fixed head dimension or an unsupported broadcast mode. + Use the input's device when selecting CPU versus CUDA: a CUDA-capable + process can still receive CPU tensors. A CPU fallback must not disable a + process-wide GPU fusion gate. +- Validate the selected implementation's inputs before launching. For JIT CUDA, + use `TensorMatcher` and `CHECK_HOST` in the C++ launcher; keep dtype checks in + the cached module factory. For Triton, validate in the Python launch wrapper. + Mismatched shapes or devices are errors, not reasons to silently try another + backend. Preserve tensor dimensions until they have been validated. +- Add a `can_use_*` predicate only when a caller needs a backend choice. A direct + kernel entry point does not need a second copy of the same predicate. Keep + fallback selection in one wrapper when several callers share it. +- Let errors from a selected implementation propagate. Do not catch every + exception and permanently disable a kernel for later requests. First-use + numerical verification in `sites/` is a separate policy: retain its reference + comparison when exactness depends on a library's reduction or rounding order. -The kernel raises on an unsupported input. It does not return `None` — a -silent `None` is too easy to forget to check, and the failure mode is a -wrong-looking image rather than an exception. +For example, Helios selects JIT CUDA RoPE on NVIDIA GPUs unless it uses TP +RMSNorm. The launcher validates the original batched Q/K and frequencies, +including shape relationships, dtype, device, layout and pair alignment. +There is no Python `can_use_helios_qk_rope` scan before the call. + +A numerical specialization query should take configuration values, not a list +of tensors. For example, `can_use_fused_rmsnorm_scale_shift(dtype, hidden)` +selects BF16 with width 2048 or 4096 because those are the FlashInfer reduction +trees the kernel reproduces. Its launcher validates weight, scale and shift. + +Existing `try_*` entry points return `None` for an unsupported specialization. +Keep that convention explicit at their call sites; do not add it to direct +kernel entry points. ## Selection matrix @@ -127,13 +150,12 @@ Several norms look interchangeable and are not. Start here. | Entry point | Backend | Contract | Applies to | |---|---|---|---| -| `residual_gate_add` | KDA (JIT CUDA) | bit-exact `residual + update * gate` | contiguous tensors, or a transposed-dense `[B, tokens, hidden]` residual/output with contiguous update and row-broadcast gate (SANA-Video) | +| `residual_gate_add` | JIT CUDA (contiguous), Triton (transposed) | bit-exact `residual + update * gate` | contiguous tensors, or a transposed-dense `[B, tokens, hidden]` residual/output with contiguous update and row-broadcast gate (SANA-Video) | -The transposed-dense path uses a shared-memory tile to read the update in -logical row-major order while keeping residual reads and output writes -coalesced in their `[B, hidden, tokens]` backing layout. Do not insert a -`.contiguous()` merely to reach the ordinary path; that restores an entire -tensor copy per residual site. +The transposed-dense path tiles along the physical stride-1 token dimension +for coalesced residual reads and output writes. The contiguous layouts use +JIT CUDA. Do not insert a `.contiguous()` to reach that path: it adds a full +tensor copy at each residual site. ### RoPE / QK-norm @@ -146,8 +168,8 @@ tensor copy per residual site. | `try_fused_qwen_qkv_epilogue` | JIT CUDA | bit-exact vs the selected BF16 chain | Qwen-Image QK RMSNorm + RoPE + joint QKV writes; SM90+ | | `fused_rope_rotate_half_bitexact` | Triton | bit-exact (elementwise only) | | `fused_complex_rope` | Triton | preserves CUDA complex64 multiply rounding for contiguous BSHD inputs; Qwen-Image 2.1 verifies its first call against eager | -| `rmsnorm_preserve_reduction` | Triton + aten | preserves the FP32 mean reduction and cast-before-weight rounding; fuses only pointwise work for contiguous FP16/BF16 inputs; Qwen-Image 2.1 verifies its first call | -| `qknorm_complex_rope_cuda` / `qknorm_complex_rope_pack_` | JIT CUDA | bit-exact vs `RMSNorm(cast_x_before_out_mul=True)` + complex64 RoPE for head_dim 128; the pack variant normalizes Q and K in place behind a prefix, copies prefix K/V (and optionally V) in the same launch, and accepts token-strided views of a packed QKV buffer; Qwen-Image 2.1 verifies its first call | +| `rmsnorm_preserve_reduction` | Triton + aten | preserves the FP32 mean reduction and cast-before-weight rounding; fuses only pointwise work for FP16/BF16 inputs and materializes strided inputs as contiguous; Qwen-Image 2.1 verifies its first call | +| `qknorm_complex_rope_cuda` / `qknorm_complex_rope_pack_` | JIT CUDA | bit-exact vs `RMSNorm(cast_x_before_out_mul=True)` + complex64 RoPE for head_dim 128; both accept token-strided views of a packed QKV buffer; the pack variant normalizes Q and K in place behind a prefix and copies prefix K/V (and optionally V) in the same launch; Qwen-Image 2.1 verifies its first call | | `fused_interleaved_rope_fp64` | JIT CUDA | bit-exact vs paired SANA-Video fp64 RoPE | | `fused_inplace_helios_qk_rope` | JIT CUDA | bit-exact paired in-place RoPE for Helios' transposed frequency layout | | `ltx2_qknorm_split_rope_cuda` | KDA (JIT CUDA) | close; **validated on B200** | @@ -218,7 +240,8 @@ inspecting model modules is its whole job. with its source revision and any JIT CUDA source files. 2. Export it from `__init__.py` (`_EXPORTS`) and register a `KernelSpec` (`_SPECS`). -3. Give it a `can_use_*` predicate; raise, don't return `None`. +3. Keep selection and validation separate as described above. Reuse an existing + wrapper when the fallback policy is shared. 4. State the numerical contract in the module docstring, including which shapes it was verified on. 5. If it is not bit-exact, gate it through `sites/`. It must mount for both diff --git a/python/sglang/kernels/ops/diffusion/__init__.py b/python/sglang/kernels/ops/diffusion/__init__.py index 70faab6b86..6fd1ce982e 100644 --- a/python/sglang/kernels/ops/diffusion/__init__.py +++ b/python/sglang/kernels/ops/diffusion/__init__.py @@ -564,7 +564,6 @@ for _op, _backend, _target, _caps, _description in _SPECS: _EXPORTS: dict[str, str] = { "can_use_fp8_rowwise": "quantization.fp8_rowwise_triton", "fp8_rowwise": "quantization.fp8_rowwise_triton", - "can_use_fused_gelu_tanh_cat": "activation.gelu_tanh_cat_jit", "fused_gelu_tanh_cat": "activation.gelu_tanh_cat_jit", "load_extension_with_recovery": "sglang.srt.utils.cpp_extension_loader", # Normalization: RMSNorm / LayerNorm / GroupNorm and their fused epilogues @@ -598,10 +597,8 @@ _EXPORTS: dict[str, str] = { "triton_one_pass_rms_norm": "norm.rmsnorm_onepass_triton", "can_use_flux2_strided_qknorm_rope": "rope.flux2_qknorm_rope_triton", "flux2_strided_qknorm_rope": "rope.flux2_qknorm_rope_triton", - "can_use_rmsnorm_preserve_reduction": "norm.rmsnorm_preserve_reduction", "rmsnorm_preserve_reduction": "norm.rmsnorm_preserve_reduction", "can_use_fused_rmsnorm_scale_shift": "norm.rmsnorm_scale_shift_bitexact", - "can_use_fused_scale_residual_rmsnorm_scale_shift": "norm.rmsnorm_scale_shift_bitexact", "fused_rmsnorm_scale_shift_bitexact": "norm.rmsnorm_scale_shift_bitexact", "fused_scale_residual_rmsnorm_scale_shift_bitexact": "norm.rmsnorm_scale_shift_bitexact", "fused_norm_scale_shift": "norm.scale_residual_norm_cutedsl", @@ -636,43 +633,32 @@ _EXPORTS: dict[str, str] = { "try_fused_scaled_residual_bf16": "modulate.scale_shift_triton", "try_fused_scaled_residual_add_exact": "modulate.scale_shift_triton", "timestep_embedding": "modulate.timestep_embedding_jit", - "can_use_fused_temb_table_slices": "modulate.wan_temb_table_slices_triton", "fused_temb_table_slices": "modulate.wan_temb_table_slices_triton", # Rotary embeddings and the QK-norm chains fused around them "try_fused_flux2_qkv_epilogue": "sglang.kernels.kda_kernels.flux2_qkv_epilogue_jit", "hunyuan_qkv_rope_pack": "rope.hunyuan_qkv_pack_triton", - "can_use_joint_qkv_cat": "layout.joint_qkv_cat_triton", "joint_qkv_cat": "layout.joint_qkv_cat_triton", "can_use_ltx2_qknorm_split_rope_cuda": "sglang.kernels.kda_kernels.ltx2_qknorm_split_rope_jit", "ltx2_qknorm_split_rope_cuda": "sglang.kernels.kda_kernels.ltx2_qknorm_split_rope_jit", "apply_ltx2_split_rotary_emb": "rope.ltx2_rotary_triton", - "can_use_ltx25_decoder_rope": "rope.ltx25_decoder_rope_jit", "fused_ltx25_decoder_rope": "rope.ltx25_decoder_rope_jit", "can_use_fused_inplace_qknorm_rope": "rope.qknorm_rope_jit", "fused_inplace_qknorm_rope": "rope.qknorm_rope_jit", "fused_qknorm_rope_pack_kv": "rope.qknorm_rope_jit", "try_fused_qwen_qkv_epilogue": "rope.qwen_qkv_epilogue_jit", - "can_use_fused_rope_rotate_half": "rope.rope_rotate_half_bitexact", "fused_rope_rotate_half_bitexact": "rope.rope_rotate_half_bitexact", - "can_use_interleaved_rope_fp64": "rope.interleaved_rope_fp64_jit", "fused_interleaved_rope_fp64": "rope.interleaved_rope_fp64_jit", - "can_use_helios_qk_rope": "rope.helios_qk_rope_jit", "fused_inplace_helios_qk_rope": "rope.helios_qk_rope_jit", "apply_rotary_embedding": "rope.rotary_triton", - "can_use_fused_complex_rope": "rope.complex_rope_triton", "fused_complex_rope": "rope.complex_rope_triton", "can_use_qknorm_complex_rope_cuda": "rope.qknorm_complex_rope_jit", - "can_use_qknorm_complex_rope_pack": "rope.qknorm_complex_rope_jit", "qknorm_complex_rope_cuda": "rope.qknorm_complex_rope_jit", "qknorm_complex_rope_pack_": "rope.qknorm_complex_rope_jit", # Tensor layout transformations fused with downstream quantization "try_flux2_token_cat_fp8": "sglang.kernels.kda_kernels.flux2_token_cat_fp8_triton", # Activation-function fusions - "can_use_fused_bias_glu": "activation.sana_conv_post_triton", - "can_use_fused_bias_silu": "activation.sana_conv_post_triton", "fused_bias_glu": "activation.sana_conv_post_triton", "fused_bias_silu": "activation.sana_conv_post_triton", - "can_use_fused_silu_mul": "activation.silu_mul_bitexact", "fused_packed_silu_mul_bitexact": "activation.silu_mul_bitexact", "fused_silu_mul_bitexact": "activation.silu_mul_bitexact", # Diffusion attention kernels @@ -757,7 +743,6 @@ _EXPORTS: dict[str, str] = { "mount_hunyuan_qknorm": "sites.hunyuan_qknorm_site", "try_hunyuan_qknorm": "sites.hunyuan_qknorm_site", "unmount_hunyuan_qknorm": "sites.hunyuan_qknorm_site", - "can_use_ltx2_rms_norm_modulate": "sites.ltx2_rmsnorm_modulate_site", "fused_ltx2_rms_norm_modulate": "sites.ltx2_rmsnorm_modulate_site", "ltx2_rms_norm_modulate_active": "sites.ltx2_rmsnorm_modulate_site", "mark_ltx2_rms_norm_modulate_site": "sites.ltx2_rmsnorm_modulate_site", diff --git a/python/sglang/kernels/ops/diffusion/activation/gelu_tanh_cat_jit.py b/python/sglang/kernels/ops/diffusion/activation/gelu_tanh_cat_jit.py index 5195686cb9..2c97de3367 100644 --- a/python/sglang/kernels/ops/diffusion/activation/gelu_tanh_cat_jit.py +++ b/python/sglang/kernels/ops/diffusion/activation/gelu_tanh_cat_jit.py @@ -24,34 +24,9 @@ def _module(): ) -def can_use_fused_gelu_tanh_cat(attn: torch.Tensor, mlp: torch.Tensor) -> bool: - return ( - not torch.is_grad_enabled() - and not torch.compiler.is_compiling() - and torch.version.hip is None - and attn.is_cuda - and attn.dtype == mlp.dtype == torch.bfloat16 - and attn.device == mlp.device - and attn.ndim >= 2 - and attn.shape[:-1] == mlp.shape[:-1] - and attn.shape[-1] > 0 - and mlp.shape[-1] > 0 - and attn.shape[-1] % 8 == mlp.shape[-1] % 8 == 0 - and attn.is_contiguous() - and mlp.is_contiguous() - and attn.numel() > 0 - and (attn.numel() + mlp.numel()) // 8 <= 2**31 - 1 - and attn.data_ptr() % 16 == mlp.data_ptr() % 16 == 0 - ) - - @register_custom_op(mutates_args=["output"]) def _gelu_tanh_cat(attn: torch.Tensor, mlp: torch.Tensor, output: torch.Tensor) -> None: - _module().gelu_tanh_cat( - attn.view(-1, attn.shape[-1]), - mlp.view(-1, mlp.shape[-1]), - output.view(-1, output.shape[-1]), - ) + _module().gelu_tanh_cat(attn, mlp, output) def fused_gelu_tanh_cat(attn: torch.Tensor, mlp: torch.Tensor) -> torch.Tensor: diff --git a/python/sglang/kernels/ops/diffusion/activation/sana_conv_post_triton.py b/python/sglang/kernels/ops/diffusion/activation/sana_conv_post_triton.py index f4b1252c25..43a6738845 100644 --- a/python/sglang/kernels/ops/diffusion/activation/sana_conv_post_triton.py +++ b/python/sglang/kernels/ops/diffusion/activation/sana_conv_post_triton.py @@ -72,31 +72,30 @@ def _bias_glu_kernel( tl.store(out_ptr + offsets, hidden * gate, mask=mask) -def _is_dense_bf16(x: torch.Tensor) -> bool: - return ( +def _validate_conv_post(x: torch.Tensor, bias: torch.Tensor | None) -> None: + if not ( x.is_cuda and x.dtype is torch.bfloat16 - and x.dim() == 4 + and x.ndim == 4 and x.numel() > 0 and (x.is_contiguous() or x.is_contiguous(memory_format=torch.channels_last)) - ) - - -def can_use_fused_bias_silu(x: torch.Tensor, bias: torch.Tensor) -> bool: - return ( - _is_dense_bf16(x) - and bias.is_cuda - and bias.dtype is x.dtype + ): + raise RuntimeError( + "Sana conv post-processing expects a dense BF16 CUDA NCHW tensor" + ) + if bias is not None and not ( + bias.dtype == x.dtype and bias.device == x.device - and bias.dim() == 1 - and bias.shape[0] == x.shape[1] + and bias.shape == (x.shape[1],) and bias.is_contiguous() - ) + ): + raise RuntimeError( + "bias must be a contiguous channel vector matching x's dtype and device" + ) def fused_bias_silu(x: torch.Tensor, bias: torch.Tensor) -> torch.Tensor: - if not can_use_fused_bias_silu(x, bias): - raise RuntimeError("unsupported input for Sana fused bias-SiLU") + _validate_conv_post(x, bias) out = torch.empty_like(x, memory_format=torch.preserve_format) with torch.cuda.device(x.device): _bias_silu_kernel[(triton.cdiv(x.numel(), 1024),)]( @@ -111,32 +110,15 @@ def fused_bias_silu(x: torch.Tensor, bias: torch.Tensor) -> torch.Tensor: return out -def can_use_fused_bias_glu(x: torch.Tensor, bias: torch.Tensor | None) -> bool: - return ( - _is_dense_bf16(x) - and x.shape[1] % 2 == 0 - and ( - bias is None - or ( - bias.is_cuda - and bias.dtype is x.dtype - and bias.device == x.device - and bias.dim() == 1 - and bias.shape[0] == x.shape[1] - and bias.is_contiguous() - ) - ) - ) - - def fused_bias_glu(x: torch.Tensor, bias: torch.Tensor | None) -> torch.Tensor: """Apply optional bias then ``hidden * silu(gate)`` along the channel axis. Pass no bias for an already biased native depthwise-convolution output: splitting that convolution's bias can change its accumulation rounding. """ - if not can_use_fused_bias_glu(x, bias): - raise RuntimeError("unsupported input for Sana fused bias-GLU") + _validate_conv_post(x, bias) + if x.shape[1] % 2: + raise RuntimeError("Sana GLU requires an even channel count") batch, double_channels, height, width = x.shape channels = double_channels // 2 channels_last = x.is_contiguous(memory_format=torch.channels_last) @@ -161,8 +143,6 @@ def fused_bias_glu(x: torch.Tensor, bias: torch.Tensor | None) -> torch.Tensor: __all__ = [ - "can_use_fused_bias_glu", - "can_use_fused_bias_silu", "fused_bias_glu", "fused_bias_silu", ] diff --git a/python/sglang/kernels/ops/diffusion/activation/silu_mul_bitexact.py b/python/sglang/kernels/ops/diffusion/activation/silu_mul_bitexact.py index 9486090836..c8cb8c25cc 100644 --- a/python/sglang/kernels/ops/diffusion/activation/silu_mul_bitexact.py +++ b/python/sglang/kernels/ops/diffusion/activation/silu_mul_bitexact.py @@ -60,20 +60,6 @@ def _packed_silu_mul_kernel( tl.store(out_ptr + row * D + cols, s * b, mask=mask) -def can_use_fused_silu_mul(a: torch.Tensor, b: torch.Tensor) -> bool: - return ( - a.dtype is torch.bfloat16 - and b.dtype is torch.bfloat16 - and a.is_cuda - and b.is_cuda - and a.device == b.device - and a.shape == b.shape - and a.is_contiguous() - and b.is_contiguous() - and a.numel() > 0 - ) - - def _fake_silu_mul(a: torch.Tensor, b: torch.Tensor) -> torch.Tensor: return torch.empty_like(a) @@ -85,6 +71,18 @@ def _fake_silu_mul(a: torch.Tensor, b: torch.Tensor) -> torch.Tensor: ) def fused_silu_mul_bitexact(a: torch.Tensor, b: torch.Tensor) -> torch.Tensor: """``silu(a) * b``, bit-exact vs the eager two-kernel chain.""" + if not ( + a.dtype is torch.bfloat16 + and b.dtype is torch.bfloat16 + and a.is_cuda + and b.is_cuda + and a.device == b.device + and a.shape == b.shape + and a.is_contiguous() + and b.is_contiguous() + and a.numel() > 0 + ): + raise RuntimeError("invalid input for fused_silu_mul_bitexact") out = torch.empty_like(a) numel = a.numel() with torch.cuda.device(a.device): diff --git a/python/sglang/kernels/ops/diffusion/attention/vdn_delta_factors_jit.py b/python/sglang/kernels/ops/diffusion/attention/vdn_delta_factors_jit.py index 4afd2b4ef3..06fbe850b1 100644 --- a/python/sglang/kernels/ops/diffusion/attention/vdn_delta_factors_jit.py +++ b/python/sglang/kernels/ops/diffusion/attention/vdn_delta_factors_jit.py @@ -65,34 +65,22 @@ def vdn_delta_factors( transition = torch.empty_like(A) injection = torch.empty_like(B) module = _jit_vdn_delta_factors_module() - module.vdn_delta_factors( - transition.view(-1, HEAD_DIM, HEAD_DIM), - injection.view(-1, HEAD_DIM, HEAD_DIM), - A.view(-1, HEAD_DIM, HEAD_DIM), - B.view(-1, HEAD_DIM, HEAD_DIM), - alpha.view(-1, HEAD_DIM), - ) + module.vdn_delta_factors(transition, injection, A, B, alpha) return transition, injection +@torch.compiler.assume_constant_result +@cache_once def can_use_vdn_delta_factors( - A: torch.Tensor, B: torch.Tensor, alpha: torch.Tensor + device: torch.device, dtype: torch.dtype, head_dim: int ) -> bool: + """The register inverse is specialized to FP32, 128-wide heads on SM80+.""" return ( - A.is_cuda - and A.dtype is torch.float32 - and B.dtype is torch.float32 - and alpha.dtype is torch.float32 - and A.device == B.device == alpha.device - and A.dim() >= 2 - and A.shape[-1] == HEAD_DIM - and A.shape[-2] == HEAD_DIM - and B.shape == A.shape - and alpha.shape == A.shape[:-1] - and A.is_contiguous() - and B.is_contiguous() - and alpha.is_contiguous() - and torch.cuda.get_device_capability(A.device)[0] >= 8 + device.type == "cuda" + and not torch.version.hip + and dtype is torch.float32 + and head_dim == HEAD_DIM + and torch.cuda.get_device_capability(device)[0] >= 8 ) diff --git a/python/sglang/kernels/ops/diffusion/layout/joint_qkv_cat_triton.py b/python/sglang/kernels/ops/diffusion/layout/joint_qkv_cat_triton.py index 0476ee6581..87a8d37c9c 100644 --- a/python/sglang/kernels/ops/diffusion/layout/joint_qkv_cat_triton.py +++ b/python/sglang/kernels/ops/diffusion/layout/joint_qkv_cat_triton.py @@ -55,35 +55,6 @@ def _joint_qkv_cat_kernel( tl.store(OUT + (component * BATCH * sequence + row) * HIDDEN + cols, value, mask) -def can_use_joint_qkv_cat(*inputs: torch.Tensor) -> bool: - if len(inputs) != 6 or torch.compiler.is_compiling() or torch.version.hip: - return False - first = inputs[0] - if first.ndim != 4 or not first.is_cuda: - return False - if first.dtype not in (torch.float16, torch.bfloat16): - return False - batch, tokens, heads, dim = first.shape - if min(batch, tokens, heads, dim) <= 0 or heads * dim > 8192: - return False - for index, value in enumerate(inputs): - expected = first.shape if index < 3 else inputs[3].shape - if ( - value.ndim != 4 - or value.shape != expected - or value.shape[0] != batch - or value.shape[2:] != (heads, dim) - or value.shape[1] <= 0 - or value.dtype != first.dtype - or value.device != first.device - or value.requires_grad - or value.stride(-1) != 1 - or value.stride(-2) != dim - ): - return False - return True - - def joint_qkv_cat( img_q: torch.Tensor, img_k: torch.Tensor, @@ -98,7 +69,33 @@ def joint_qkv_cat( The three outputs occupy disjoint, contiguous regions of one allocation. """ inputs = (img_q, img_k, img_v, txt_q, txt_k, txt_v) - assert can_use_joint_qkv_cat(*inputs) + if not ( + img_q.is_cuda + and img_q.dtype in (torch.float16, torch.bfloat16) + and img_q.ndim == 4 + and min(img_q.shape) > 0 + and img_q.shape[-2] * img_q.shape[-1] <= 8192 + ): + raise RuntimeError( + "joint QKV expects FP16/BF16 CUDA [B, S, H, D], 0 < H * D <= 8192" + ) + for index, value in enumerate(inputs): + expected = img_q.shape if index < 3 else txt_q.shape + if not ( + value.ndim == 4 + and value.shape == expected + and value.shape[0] == img_q.shape[0] + and value.shape[2:] == img_q.shape[2:] + and value.shape[1] > 0 + and value.dtype == img_q.dtype + and value.device == img_q.device + and not value.requires_grad + and value.stride(-1) == 1 + and value.stride(-2) == img_q.shape[-1] + ): + raise RuntimeError( + "joint QKV inputs must share batch/head shape, dtype/device and have packed head rows without gradients" + ) batch, image_tokens, heads, dim = img_q.shape text_tokens = txt_q.shape[1] output = torch.empty( diff --git a/python/sglang/kernels/ops/diffusion/layout/usp_relayout_jit.py b/python/sglang/kernels/ops/diffusion/layout/usp_relayout_jit.py index 816706a9b5..6c73d344c2 100644 --- a/python/sglang/kernels/ops/diffusion/layout/usp_relayout_jit.py +++ b/python/sglang/kernels/ops/diffusion/layout/usp_relayout_jit.py @@ -42,7 +42,7 @@ def _fake_merge_heads(x: torch.Tensor) -> torch.Tensor: mutates_args=[], fake_impl=_fake_merge_heads, ) -def _usp_merge_heads_custom_op(x: torch.Tensor) -> torch.Tensor: +def _usp_merge_heads_cuda(x: torch.Tensor) -> torch.Tensor: world, seq, batch, h_local, head_dim = x.shape out = x.new_empty((batch, seq, world, h_local, head_dim)) module = _jit_usp_relayout_module(x.dtype) @@ -52,27 +52,14 @@ def _usp_merge_heads_custom_op(x: torch.Tensor) -> torch.Tensor: def can_use_usp_merge_heads(x: torch.Tensor) -> bool: return ( - isinstance(x, torch.Tensor) - and torch.version.hip is None + torch.version.hip is None and x.is_cuda and x.dtype in _SUPPORTED_DTYPES - and x.dim() == 5 and x.numel() > 0 and x.is_contiguous() ) -def _usp_merge_heads_cuda(x: torch.Tensor) -> torch.Tensor: - """[W, S, B, h_local, D] -> [B, S, W, h_local, D] contiguous. - - Bit-exact single-pass replacement for - ``x.permute(2, 1, 0, 3, 4).contiguous()`` on the Ulysses output path. - """ - if not can_use_usp_merge_heads(x): - raise RuntimeError("unsupported input for usp_merge_heads CUDA") - return _usp_merge_heads_custom_op(x) - - def usp_merge_heads(x: torch.Tensor) -> torch.Tensor: """Merge Ulysses output heads with an exact eager fallback. diff --git a/python/sglang/kernels/ops/diffusion/modulate/modulate_scale_shift_jit.py b/python/sglang/kernels/ops/diffusion/modulate/modulate_scale_shift_jit.py index 7c4d59b994..1f13f7bf07 100644 --- a/python/sglang/kernels/ops/diffusion/modulate/modulate_scale_shift_jit.py +++ b/python/sglang/kernels/ops/diffusion/modulate/modulate_scale_shift_jit.py @@ -7,7 +7,6 @@ bit-exact vs the eager chain (``torch.equal``) and needs no quality gate. from __future__ import annotations -import logging from typing import TYPE_CHECKING import torch @@ -21,9 +20,6 @@ if TYPE_CHECKING: _SUPPORTED_DTYPES = (torch.float16, torch.bfloat16) _ALIGN_BYTES = 16 -_FAILED_RUNTIME_KEYS: set[tuple[int | None, torch.dtype]] = set() - -logger = logging.getLogger(__name__) @cache_once @@ -55,9 +51,10 @@ def _fake_impl( mutates_args=[], fake_impl=_fake_impl, ) -def _modulate_scale_shift_custom_op( +def modulate_scale_shift_cuda( x: torch.Tensor, scale: torch.Tensor, shift: torch.Tensor ) -> torch.Tensor: + """Fused ``x * (1 + scale[:, None]) + shift[:, None]``; validated by CUDA.""" out = torch.empty_like(x) module = _jit_modulate_scale_shift_module(x.dtype) module.modulate_scale_shift(out, x, scale, shift) @@ -72,7 +69,8 @@ def can_use_modulate_scale_shift_cuda( x: torch.Tensor, scale: torch.Tensor, shift: torch.Tensor ) -> bool: if ( - x.dtype not in _SUPPORTED_DTYPES + torch.version.hip is not None + or x.dtype not in _SUPPORTED_DTYPES or scale.dtype != x.dtype or shift.dtype != x.dtype or not (x.is_cuda and scale.is_cuda and shift.is_cuda) @@ -91,35 +89,12 @@ def can_use_modulate_scale_shift_cuda( ) -def modulate_scale_shift_cuda( - x: torch.Tensor, scale: torch.Tensor, shift: torch.Tensor -) -> torch.Tensor: - """Fused ``x * (1 + scale[:, None]) + shift[:, None]`` (bit-exact vs eager).""" - if not can_use_modulate_scale_shift_cuda(x, scale, shift): - raise RuntimeError("unsupported input for modulate_scale_shift CUDA") - return _modulate_scale_shift_custom_op(x, scale, shift) - - def modulate_scale_shift( x: torch.Tensor, scale: torch.Tensor, shift: torch.Tensor ) -> torch.Tensor: """Use the bit-exact CUDA fast path when supported, otherwise eager.""" - runtime_key = (x.device.index, x.dtype) - if runtime_key not in _FAILED_RUNTIME_KEYS and can_use_modulate_scale_shift_cuda( - x, scale, shift - ): - try: - return modulate_scale_shift_cuda(x, scale, shift) - except Exception as exc: - if torch.compiler.is_compiling(): - raise - _FAILED_RUNTIME_KEYS.add(runtime_key) - logger.warning( - "Disabling diffusion modulate CUDA fast path on %s/%s: %s", - x.device, - x.dtype, - exc, - ) + if can_use_modulate_scale_shift_cuda(x, scale, shift): + return modulate_scale_shift_cuda(x, scale, shift) return x * (1 + scale[:, None]) + shift[:, None] diff --git a/python/sglang/kernels/ops/diffusion/modulate/wan_temb_table_slices_triton.py b/python/sglang/kernels/ops/diffusion/modulate/wan_temb_table_slices_triton.py index b031b894c5..ecd8159179 100644 --- a/python/sglang/kernels/ops/diffusion/modulate/wan_temb_table_slices_triton.py +++ b/python/sglang/kernels/ops/diffusion/modulate/wan_temb_table_slices_triton.py @@ -49,22 +49,6 @@ def _temb_table_slices_kernel( tl.store(out_ptr + (j * rows + row) * D + cols, w + t, mask=mask) -def can_use_fused_temb_table_slices(table: torch.Tensor, temb: torch.Tensor) -> bool: - return ( - temb.is_cuda - and temb.dtype in (torch.bfloat16, torch.float16, torch.float32) - and temb.dim() == 4 - and temb.shape[2] == 6 - and temb.is_contiguous() - and table.is_cuda - and table.device == temb.device - and table.dtype in (torch.bfloat16, torch.float16, torch.float32) - and table.shape == (1, 6, temb.shape[-1]) - and table.is_contiguous() - and temb.numel() > 0 - ) - - def _fake_temb_table_slices(table: torch.Tensor, temb: torch.Tensor) -> torch.Tensor: batch, seq_len, six, hidden = temb.shape return temb.new_empty((six, batch, seq_len, hidden), dtype=torch.float32) @@ -81,6 +65,20 @@ def fused_temb_table_slices(table: torch.Tensor, temb: torch.Tensor) -> torch.Te ``temb`` is ``(B, S, 6, D)``; ``table`` is the block's ``(1, 6, D)`` fp32 adaLN table. ``out[j]`` is the ``j``-th modulation slice, contiguous. """ + if not ( + temb.is_cuda + and temb.dtype in (torch.bfloat16, torch.float16, torch.float32) + and temb.dim() == 4 + and temb.shape[2] == 6 + and temb.is_contiguous() + and table.is_cuda + and table.device == temb.device + and table.dtype in (torch.bfloat16, torch.float16, torch.float32) + and table.shape == (1, 6, temb.shape[-1]) + and table.is_contiguous() + and temb.numel() > 0 + ): + raise RuntimeError("invalid input for fused_temb_table_slices") batch, seq_len, _, hidden = temb.shape out = temb.new_empty((6, batch, seq_len, hidden), dtype=torch.float32) rows = batch * seq_len diff --git a/python/sglang/kernels/ops/diffusion/norm/rmsnorm_preserve_reduction.py b/python/sglang/kernels/ops/diffusion/norm/rmsnorm_preserve_reduction.py index 7c573980b6..37c77dc556 100644 --- a/python/sglang/kernels/ops/diffusion/norm/rmsnorm_preserve_reduction.py +++ b/python/sglang/kernels/ops/diffusion/norm/rmsnorm_preserve_reduction.py @@ -2,7 +2,7 @@ """Fuse RMSNorm pointwise work while retaining the native FP32 mean reduction. Matches ``weight * (x.float() * rsqrt(mean(x.float()**2) + eps)).to(x.dtype)`` -for contiguous FP16/BF16 inputs. The FP32 square buffer has the original shape, +for FP16/BF16 inputs; packed projection views are materialized before the launch. The FP32 square buffer has the original shape, so aten selects the same reduction as the eager chain. Verified at head width 128, including 131072 rows; callers verify their first dispatch before reuse. """ @@ -42,23 +42,8 @@ def _rmsnorm_finish_kernel( tl.store(out_ptr + index, normalized.to(tl.float32) * weight, mask) -def can_use_rmsnorm_preserve_reduction(x: torch.Tensor, weight: torch.Tensor) -> bool: - return ( - x.is_cuda - and torch.version.hip is None - and x.dtype in (torch.float16, torch.bfloat16) - and x.ndim >= 2 - and x.numel() > 0 - and x.is_contiguous() - and weight.device == x.device - and weight.dtype == x.dtype - and weight.shape == (x.shape[-1],) - and weight.is_contiguous() - ) - - def _fake_rmsnorm(x: torch.Tensor, weight: torch.Tensor, eps: float) -> torch.Tensor: - return torch.empty_like(x) + return torch.empty(x.shape, dtype=x.dtype, device=x.device) @register_custom_op( @@ -70,7 +55,19 @@ def rmsnorm_preserve_reduction( x: torch.Tensor, weight: torch.Tensor, eps: float ) -> torch.Tensor: """Preserve aten's mean and cast-before-weight semantics without residuals.""" - assert can_use_rmsnorm_preserve_reduction(x, weight) + if not ( + x.is_cuda + and torch.version.hip is None + and x.dtype in (torch.float16, torch.bfloat16) + and x.ndim >= 2 + and x.numel() > 0 + and weight.device == x.device + and weight.dtype == x.dtype + and weight.shape == (x.shape[-1],) + and weight.is_contiguous() + ): + raise RuntimeError("invalid input for rmsnorm_preserve_reduction") + x = x.contiguous() squares = torch.empty_like(x, dtype=torch.float32) out = torch.empty_like(x) with torch.cuda.device(x.device): diff --git a/python/sglang/kernels/ops/diffusion/norm/rmsnorm_scale_shift_bitexact.py b/python/sglang/kernels/ops/diffusion/norm/rmsnorm_scale_shift_bitexact.py index 4fc7b8f9a4..7b85c65183 100644 --- a/python/sglang/kernels/ops/diffusion/norm/rmsnorm_scale_shift_bitexact.py +++ b/python/sglang/kernels/ops/diffusion/norm/rmsnorm_scale_shift_bitexact.py @@ -155,59 +155,57 @@ def _threads_per_row(hidden: int) -> int | None: return tpr -def _is_row_broadcast(t: torch.Tensor, x: torch.Tensor) -> bool: +def _is_row_broadcast(t: torch.Tensor, shape: tuple, device: torch.device) -> bool: return ( t.dtype is torch.bfloat16 - and t.is_cuda - and t.device == x.device - and t.shape == (x.shape[0], 1, x.shape[-1]) + and t.device == device + and t.shape == shape and t.is_contiguous() ) -def can_use_fused_rmsnorm_scale_shift( - x: torch.Tensor, - weight: torch.Tensor, - scale: torch.Tensor, - shift: torch.Tensor, -) -> bool: +def can_use_fused_rmsnorm_scale_shift(dtype: torch.dtype, hidden: int) -> bool: + """Select the BF16 FlashInfer reduction replicated by this kernel.""" return ( - # ROCm cannot compile the inline PTX above: LLVM makes the unusable - # `=f` constraint a fatal error that kills the process, so reject - # before the first launch rather than rely on the caller's fallback. + is_cuda() and dtype is torch.bfloat16 and _threads_per_row(hidden) is not None + ) + + +def _validate_rmsnorm_modulate(x, weight, scale, shift) -> tuple[int, int, int, int]: + shape = x.shape + if not ( is_cuda() - and x.dtype is torch.bfloat16 and x.is_cuda - and x.dim() == 3 + and x.dtype is torch.bfloat16 + and len(shape) == 3 and x.is_contiguous() - and _threads_per_row(x.shape[-1]) is not None - and weight.dtype is torch.bfloat16 - and weight.is_cuda - and weight.device == x.device - and weight.shape == (x.shape[-1],) + ): + raise RuntimeError( + "RMSNorm modulation expects contiguous BF16 CUDA [B, S, D], D in (2048, 4096)" + ) + batch, seq_len, hidden = shape + tpr = _threads_per_row(hidden) + if tpr is None: + raise RuntimeError("RMSNorm modulation requires D in (2048, 4096)") + device = x.device + if not ( + weight.dtype is torch.bfloat16 + and weight.device == device + and weight.shape == (hidden,) and weight.is_contiguous() - and _is_row_broadcast(scale, x) - and _is_row_broadcast(shift, x) - ) - - -def can_use_fused_scale_residual_rmsnorm_scale_shift( - residual: torch.Tensor, - update: torch.Tensor, - gate: torch.Tensor, - weight: torch.Tensor, - scale: torch.Tensor, - shift: torch.Tensor, -) -> bool: - return ( - can_use_fused_rmsnorm_scale_shift(residual, weight, scale, shift) - and update.dtype is torch.bfloat16 - and update.is_cuda - and update.device == residual.device - and update.shape == residual.shape - and update.is_contiguous() - and _is_row_broadcast(gate, residual) - ) + ): + raise RuntimeError( + "weight must be a contiguous [D] vector matching x's dtype/device" + ) + row_shape = (batch, 1, hidden) + if not ( + _is_row_broadcast(scale, row_shape, device) + and _is_row_broadcast(shift, row_shape, device) + ): + raise RuntimeError( + "scale and shift must be contiguous [B, 1, D] tensors matching x's dtype/device" + ) + return batch, seq_len, hidden, tpr def _fake_norm_scale_shift( @@ -233,8 +231,7 @@ def fused_rmsnorm_scale_shift_bitexact( eps: float, ) -> torch.Tensor: """``norm(x) * (1 + scale) + shift``, bit-exact vs the eager chain.""" - batch, seq_len, hidden = x.shape - tpr = _threads_per_row(hidden) + batch, seq_len, hidden, tpr = _validate_rmsnorm_modulate(x, weight, scale, shift) out = torch.empty_like(x) with torch.cuda.device(x.device): _rmsnorm_scale_shift_kernel[(batch * seq_len,)]( @@ -291,8 +288,19 @@ def fused_scale_residual_rmsnorm_scale_shift_bitexact( Returns ``(modulated, res)``, both bit-exact vs the eager pair + norm chain (and therefore vs the ``residual_gate_add_cuda`` fast path). """ - batch, seq_len, hidden = residual.shape - tpr = _threads_per_row(hidden) + batch, seq_len, hidden, tpr = _validate_rmsnorm_modulate( + residual, weight, scale, shift + ) + if not ( + update.dtype == residual.dtype + and update.device == residual.device + and update.shape == residual.shape + and update.is_contiguous() + and _is_row_broadcast(gate, (batch, 1, hidden), residual.device) + ): + raise RuntimeError( + "update must match residual; gate must be contiguous [B, 1, D] with the same dtype/device" + ) out = torch.empty_like(residual) res_out = torch.empty_like(residual) with torch.cuda.device(residual.device): diff --git a/python/sglang/kernels/ops/diffusion/rope/complex_rope_triton.py b/python/sglang/kernels/ops/diffusion/rope/complex_rope_triton.py index 1cdae680c8..68d0d9a5e6 100644 --- a/python/sglang/kernels/ops/diffusion/rope/complex_rope_triton.py +++ b/python/sglang/kernels/ops/diffusion/rope/complex_rope_triton.py @@ -50,22 +50,6 @@ def _fuse_real_sin(device: torch.device) -> bool: return (z[0] * z[1]).imag.item() != 0 -def can_use_fused_complex_rope(x: torch.Tensor, rope: torch.Tensor) -> bool: - return ( - x.is_cuda - and torch.version.hip is None - and x.dtype in (torch.float16, torch.bfloat16, torch.float32) - and x.ndim == 4 - and x.numel() > 0 - and x.shape[-1] % 2 == 0 - and x.is_contiguous() - and rope.dtype == torch.complex64 - and rope.device == x.device - and rope.shape == (x.shape[1], x.shape[-1] // 2) - and rope.is_contiguous() - ) - - def _fake_complex_rope(x: torch.Tensor, rope: torch.Tensor) -> torch.Tensor: return torch.empty_like(x) @@ -77,7 +61,20 @@ def _fake_complex_rope(x: torch.Tensor, rope: torch.Tensor) -> torch.Tensor: ) def fused_complex_rope(x: torch.Tensor, rope: torch.Tensor) -> torch.Tensor: """Rotate contiguous BSHD activations with a shared S×(D/2) complex cache.""" - assert can_use_fused_complex_rope(x, rope) + if not ( + x.is_cuda + and torch.version.hip is None + and x.dtype in (torch.float16, torch.bfloat16, torch.float32) + and x.ndim == 4 + and x.numel() > 0 + and x.shape[-1] % 2 == 0 + and x.is_contiguous() + and rope.dtype == torch.complex64 + and rope.device == x.device + and rope.shape == (x.shape[1], x.shape[-1] // 2) + and rope.is_contiguous() + ): + raise RuntimeError("invalid input for fused_complex_rope") out = torch.empty_like(x) pairs = x.numel() // 2 with torch.cuda.device(x.device): diff --git a/python/sglang/kernels/ops/diffusion/rope/helios_qk_rope_jit.py b/python/sglang/kernels/ops/diffusion/rope/helios_qk_rope_jit.py index e5f4b72d68..d0cc460c5f 100644 --- a/python/sglang/kernels/ops/diffusion/rope/helios_qk_rope_jit.py +++ b/python/sglang/kernels/ops/diffusion/rope/helios_qk_rope_jit.py @@ -1,11 +1,4 @@ -"""Bit-exact paired RoPE for Helios' transposed frequency layout. - -The JIT kernel preserves the eager path's separate fp32 multiply and add/sub -rounding boundaries, then rounds each adjacent pair back to fp16/bf16 in -place. It is verified on head dimensions 64, 128, and 256, including Helios' -production ``[8640, 40, 128]`` Q/K shape. Unsupported layouts retain the eager -model path through :func:`can_use_helios_qk_rope`. -""" +"""Helios paired RoPE, preserving separate FP32 multiply/add rounding.""" from __future__ import annotations @@ -22,10 +15,8 @@ if TYPE_CHECKING: @cache_once def _jit_helios_qk_rope_module(dtype: torch.dtype) -> Module: - if dtype not in (torch.float16, torch.bfloat16): - raise RuntimeError( - f"Unsupported Helios QK RoPE dtype {dtype}; expected float16 or bfloat16" - ) + if dtype not in (torch.float16, torch.bfloat16, torch.float32): + raise RuntimeError(f"Unsupported Helios QK RoPE dtype: {dtype}") args = make_cpp_args(dtype) return load_jit( "helios_qk_rope", @@ -41,41 +32,13 @@ def fused_inplace_helios_qk_rope( k: torch.Tensor, freqs: torch.Tensor, ) -> None: - """Apply Helios' transposed RoPE to contiguous normalized Q/K in place.""" + """Rotate contiguous Q/K shaped [N, H, D] or [B, S, H, D] in place. + + Frequencies have shape [N, 2D] or [B, S, 2D]. The C++ launcher + validates these shapes, dtypes, devices and pair alignment. + """ module = _jit_helios_qk_rope_module(q.dtype) module.helios_qk_rope(q, k, freqs) -def can_use_helios_qk_rope( - q: torch.Tensor, - k: torch.Tensor, - freqs: torch.Tensor, -) -> bool: - """Return whether tensors match the native Helios paired-RoPE contract.""" - if q.dim() != 4 or freqs.dim() != 3: - return False - # Dynamo cannot trace pointer or storage-offset queries. Compiled Helios Q/K - # come directly from aligned linear outputs; eager callers retain the guard. - pair_aligned = True - if not torch.compiler.is_compiling(): - pair_aligned = q.storage_offset() % 2 == 0 and k.storage_offset() % 2 == 0 - return ( - q.is_cuda - and k.is_cuda - and freqs.is_cuda - and q.dtype in (torch.float16, torch.bfloat16) - and k.dtype == q.dtype - and freqs.dtype is torch.float32 - and q.device == k.device == freqs.device - and k.shape == q.shape - and all(size > 0 for size in q.shape) - and freqs.shape == (*q.shape[:2], 2 * q.shape[-1]) - and q.shape[-1] % 2 == 0 - and q.is_contiguous() - and k.is_contiguous() - and freqs.is_contiguous() - and pair_aligned - ) - - -__all__ = ["can_use_helios_qk_rope", "fused_inplace_helios_qk_rope"] +__all__ = ["fused_inplace_helios_qk_rope"] diff --git a/python/sglang/kernels/ops/diffusion/rope/interleaved_rope_fp64_jit.py b/python/sglang/kernels/ops/diffusion/rope/interleaved_rope_fp64_jit.py index ebf55a8df4..5bf1258f4b 100644 --- a/python/sglang/kernels/ops/diffusion/rope/interleaved_rope_fp64_jit.py +++ b/python/sglang/kernels/ops/diffusion/rope/interleaved_rope_fp64_jit.py @@ -54,52 +54,10 @@ def fused_interleaved_rope_fp64( q_out = torch.empty_like(q) k_out = torch.empty_like(k) module = _jit_interleaved_rope_fp64_module(q.dtype) - module.interleaved_rope_fp64( - q_out.view(-1), - k_out.view(-1), - q.view(-1), - k.view(-1), - cos.view(-1), - sin.view(-1), - q.shape[0], - q.shape[1], - q.shape[2], - q.shape[3], - ) + module.interleaved_rope_fp64(q_out, k_out, q, k, cos, sin) return q_out, k_out -def can_use_interleaved_rope_fp64( - q: torch.Tensor, - k: torch.Tensor, - cos: torch.Tensor, - sin: torch.Tensor, -) -> bool: - if q.dim() != 4: - return False - expected_table_shape = (1, q.shape[1], 1, q.shape[3]) - return ( - q.dtype is torch.bfloat16 - and k.dtype is q.dtype - and q.is_cuda - and k.is_cuda - and q.device == k.device == cos.device == sin.device - and k.shape == q.shape - and q.shape[-1] % 2 == 0 - and q.is_contiguous() - and k.is_contiguous() - and q.data_ptr() % 4 == 0 - and k.data_ptr() % 4 == 0 - and cos.dtype is torch.float64 - and sin.dtype is torch.float64 - and cos.shape == expected_table_shape - and sin.shape == expected_table_shape - and cos.is_contiguous() - and sin.is_contiguous() - ) - - __all__ = [ - "can_use_interleaved_rope_fp64", "fused_interleaved_rope_fp64", ] diff --git a/python/sglang/kernels/ops/diffusion/rope/ltx25_decoder_rope_jit.py b/python/sglang/kernels/ops/diffusion/rope/ltx25_decoder_rope_jit.py index f2a714d579..3507e9431d 100644 --- a/python/sglang/kernels/ops/diffusion/rope/ltx25_decoder_rope_jit.py +++ b/python/sglang/kernels/ops/diffusion/rope/ltx25_decoder_rope_jit.py @@ -67,74 +67,11 @@ def fused_ltx25_decoder_rope( k_out = torch.empty_like(k) module = _jit_ltx25_decoder_rope_module(q.dtype) module.ltx25_decoder_rope( - q_out.view(-1), - k_out.view(-1), - q.view(-1), - k.view(-1), - cos_t.view(-1), - sin_t.view(-1), - cos_h.view(-1), - sin_h.view(-1), - cos_w.view(-1), - sin_w.view(-1), - q.shape[0], - q.shape[1], - q.shape[2], - q.shape[3], - q.shape[4], - q.shape[5], - dim_t, - dim_h, + q_out, k_out, q, k, cos_t, sin_t, cos_h, sin_h, cos_w, sin_w, dim_t, dim_h ) return q_out, k_out -def can_use_ltx25_decoder_rope( - q: torch.Tensor, - k: torch.Tensor, - tables: tuple[tuple[torch.Tensor, torch.Tensor], ...], - dim_split: tuple[int, int, int], -) -> bool: - if ( - q.dim() != 6 - or len(tables) != 3 - or any(len(pair) != 2 for pair in tables) - or len(dim_split) != 3 - ): - return False - dim_t, dim_h, dim_w = dim_split - expected_shapes = ( - (q.shape[1], dim_t // 2), - (q.shape[2], dim_h // 2), - (q.shape[3], dim_w // 2), - ) - flat_tables = tuple(table for pair in tables for table in pair) - return ( - q.dtype is torch.bfloat16 - and k.dtype is q.dtype - and q.is_cuda - and k.is_cuda - and q.device == k.device - and k.shape == q.shape - and all(size > 0 for size in q.shape) - and q.shape[-1] == sum(dim_split) - and all(dim > 0 and dim % 2 == 0 for dim in dim_split) - and q.is_contiguous() - and k.is_contiguous() - and q.data_ptr() % 4 == 0 - and k.data_ptr() % 4 == 0 - and all(table.device == q.device for table in flat_tables) - and all(table.dtype is torch.float32 for table in flat_tables) - and all( - cos.shape == sin.shape == expected_shape - and cos.is_contiguous() - and sin.is_contiguous() - for (cos, sin), expected_shape in zip(tables, expected_shapes, strict=True) - ) - ) - - __all__ = [ - "can_use_ltx25_decoder_rope", "fused_ltx25_decoder_rope", ] diff --git a/python/sglang/kernels/ops/diffusion/rope/qknorm_complex_rope_jit.py b/python/sglang/kernels/ops/diffusion/rope/qknorm_complex_rope_jit.py index f30d0932f6..12747bae49 100644 --- a/python/sglang/kernels/ops/diffusion/rope/qknorm_complex_rope_jit.py +++ b/python/sglang/kernels/ops/diffusion/rope/qknorm_complex_rope_jit.py @@ -33,7 +33,6 @@ if TYPE_CHECKING: from tvm_ffi.module import Module _HEAD_DIM = 128 -_ROW_ALIGN = 8 @cache_once @@ -50,117 +49,19 @@ def _jit_module(fuse_real_sin: bool, copy_v: bool) -> Module: ) -def _is_rows(t: torch.Tensor, device: torch.device, tokens: int | None = None) -> bool: - """``[B, T, H, 128]`` bf16 CUDA tensor whose heads are contiguous; batch/token strides may be padded.""" - return ( - isinstance(t, torch.Tensor) - and t.dtype is torch.bfloat16 - and t.is_cuda - and t.device == device - and t.dim() == 4 - and t.numel() > 0 - and t.shape[-1] == _HEAD_DIM - and (tokens is None or t.shape[1] == tokens) - and t.stride(-1) == 1 - and t.stride(-2) == _HEAD_DIM - and t.stride(1) % (_ROW_ALIGN // 2) == 0 - and t.stride(0) % (_ROW_ALIGN // 2) == 0 - and t.data_ptr() % _ROW_ALIGN == 0 - ) - - -def _is_weight(w: torch.Tensor, device: torch.device) -> bool: - return ( - isinstance(w, torch.Tensor) - and w.dtype is torch.bfloat16 - and w.is_cuda - and w.device == device - and w.shape == (_HEAD_DIM,) - and w.is_contiguous() - ) - - -def _is_rope(rope: torch.Tensor, seq: int, device: torch.device) -> bool: - return ( - isinstance(rope, torch.Tensor) - and rope.dtype is torch.complex64 - and rope.device == device - and rope.shape == (seq, _HEAD_DIM // 2) - and rope.is_contiguous() - ) - - -def _cuda_platform(t: torch.Tensor) -> bool: - return t.is_cuda and torch.version.hip is None and not torch.compiler.is_compiling() +@torch.compiler.assume_constant_result +@cache_once +def can_use_qknorm_complex_rope_cuda(dtype: torch.dtype, head_dim: int) -> bool: + """The CUDA reduction is specialized to BF16, 128-wide heads.""" + return dtype is torch.bfloat16 and head_dim == _HEAD_DIM def _rope_real(rope: torch.Tensor) -> torch.Tensor: - return torch.view_as_real(rope).view(rope.shape[0], _HEAD_DIM) - - -def can_use_qknorm_complex_rope_cuda( - x: torch.Tensor, weight: torch.Tensor, rope: torch.Tensor -) -> bool: - """Contiguous ``[B, S, H, 128]`` bf16 Q or K with its ``(S, 64)`` complex64 table.""" - return ( - _cuda_platform(x) - and _is_rows(x, x.device) - and x.is_contiguous() - and _is_weight(weight, x.device) - and _is_rope(rope, x.shape[1], x.device) - ) - - -def can_use_qknorm_complex_rope_pack( - q: torch.Tensor, - k_out: torch.Tensor, - v_out: torch.Tensor, - q_weight: torch.Tensor, - k_weight: torch.Tensor, - rope: torch.Tensor, - k_prefix: torch.Tensor, - v_prefix: torch.Tensor, - k_src: Optional[torch.Tensor], - v_src: Optional[torch.Tensor], -) -> bool: - """Shapes for :func:`qknorm_complex_rope_pack_`; ``None`` sources mean "already in the buffer".""" - if not (_cuda_platform(q) and _is_rows(q, q.device)): - return False - device = q.device - batch, seq, heads, _ = q.shape - if not ( - k_prefix.dim() == 4 - and k_prefix.shape[0] == batch - and k_prefix.shape[1] > 0 - and k_prefix.shape[2] == heads - and k_prefix.shape[3] == _HEAD_DIM - and k_prefix.is_contiguous() - and v_prefix.shape == k_prefix.shape - and v_prefix.is_contiguous() - and all( - t.dtype is torch.bfloat16 and t.device == device - for t in (k_prefix, v_prefix) - ) - ): - return False - prefix = k_prefix.shape[1] - return ( - _is_rows(k_out, device, prefix + seq) - and _is_rows(v_out, device, prefix + seq) - and k_out.shape[0] == batch - and v_out.shape[0] == batch - and k_out.shape[2] == heads - and v_out.shape[2] == heads - and (k_src is None or (_is_rows(k_src, device, seq) and k_src.shape == q.shape)) - and (v_src is None or (_is_rows(v_src, device, seq) and v_src.shape == q.shape)) - and _is_weight(q_weight, device) - and _is_weight(k_weight, device) - and _is_rope(rope, seq, device) - ) + return torch.view_as_real(rope) def _fake_norm_rope(x, weight, rope, eps): - return torch.empty_like(x) + return torch.empty(x.shape, dtype=x.dtype, device=x.device) @register_custom_op( @@ -171,8 +72,8 @@ def _fake_norm_rope(x, weight, rope, eps): def qknorm_complex_rope_cuda( x: torch.Tensor, weight: torch.Tensor, rope: torch.Tensor, eps: float ) -> torch.Tensor: - """``rope(rmsnorm(x))`` for one contiguous ``[B, S, H, 128]`` tensor, out of place.""" - out = torch.empty_like(x) + """``rope(rmsnorm(x))`` for one ``[B, S, H, 128]`` tensor with packed head rows, out of place.""" + out = torch.empty(x.shape, dtype=x.dtype, device=x.device) module = _jit_module(_fuse_real_sin(x.device), False) with torch.cuda.device(x.device): module.norm_rope(x, out, weight, _rope_real(rope), float(eps)) @@ -224,7 +125,6 @@ def qknorm_complex_rope_pack_( __all__ = [ "can_use_qknorm_complex_rope_cuda", - "can_use_qknorm_complex_rope_pack", "qknorm_complex_rope_cuda", "qknorm_complex_rope_pack_", ] diff --git a/python/sglang/kernels/ops/diffusion/rope/qknorm_complex_rope_kv_triton.py b/python/sglang/kernels/ops/diffusion/rope/qknorm_complex_rope_kv_triton.py index 5b6462ab75..94c85f1d78 100644 --- a/python/sglang/kernels/ops/diffusion/rope/qknorm_complex_rope_kv_triton.py +++ b/python/sglang/kernels/ops/diffusion/rope/qknorm_complex_rope_kv_triton.py @@ -8,7 +8,7 @@ import triton.language as tl from sglang.kernels.ops.diffusion.rope.complex_rope_triton import _fuse_real_sin from sglang.kernels.ops.diffusion.rope.qknorm_complex_rope_triton import ( _qknorm_complex_rope_rows, - can_use_qknorm_complex_rope, + _validate_qknorm_complex_rope, ) from sglang.srt.utils.custom_op import register_custom_op @@ -54,22 +54,6 @@ def _qknorm_complex_rope_kv_kernel( tl.store(vout_ptr + prefix_index, prefix_value, prefix_mask) -def can_use_qknorm_complex_rope_kv(k, weight, rope, v, k_prefix, v_prefix): - return ( - can_use_qknorm_complex_rope(k, weight, rope) - and v.shape == k.shape - and k_prefix.ndim == 4 - and k_prefix.shape[0] == k.shape[0] - and k_prefix.shape[1] > 0 - and k_prefix.shape[2:] == k.shape[2:] - and v_prefix.shape == k_prefix.shape - and all( - x.device == k.device and x.dtype == k.dtype and x.is_contiguous() - for x in (v, k_prefix, v_prefix) - ) - ) - - def _fake_qknorm_complex_rope_kv(k, weight, rope, v, k_prefix, v_prefix, eps): shape = (k.shape[0], k_prefix.shape[1] + k.shape[1], *k.shape[2:]) return k.new_empty(shape), v.new_empty(shape) @@ -89,7 +73,21 @@ def qknorm_complex_rope_kv( v_prefix: torch.Tensor, eps: float, ) -> tuple[torch.Tensor, torch.Tensor]: - assert can_use_qknorm_complex_rope_kv(k, weight, rope, v, k_prefix, v_prefix) + k, v = k.contiguous(), v.contiguous() + _validate_qknorm_complex_rope(k, weight, rope) + if not ( + v.shape == k.shape + and k_prefix.ndim == 4 + and k_prefix.shape[0] == k.shape[0] + and k_prefix.shape[1] > 0 + and k_prefix.shape[2:] == k.shape[2:] + and v_prefix.shape == k_prefix.shape + and all( + x.device == k.device and x.dtype == k.dtype and x.is_contiguous() + for x in (v, k_prefix, v_prefix) + ) + ): + raise RuntimeError("invalid input for qknorm_complex_rope_kv") kout, vout = _fake_qknorm_complex_rope_kv( k, weight, rope, v, k_prefix, v_prefix, eps ) diff --git a/python/sglang/kernels/ops/diffusion/rope/qknorm_complex_rope_triton.py b/python/sglang/kernels/ops/diffusion/rope/qknorm_complex_rope_triton.py index d3fd8cb521..4091027592 100644 --- a/python/sglang/kernels/ops/diffusion/rope/qknorm_complex_rope_triton.py +++ b/python/sglang/kernels/ops/diffusion/rope/qknorm_complex_rope_triton.py @@ -5,12 +5,8 @@ import torch import triton import triton.language as tl -from sglang.kernels.ops.diffusion.norm.rmsnorm_preserve_reduction import ( - can_use_rmsnorm_preserve_reduction, -) from sglang.kernels.ops.diffusion.rope.complex_rope_triton import ( _fuse_real_sin, - can_use_fused_complex_rope, ) from sglang.srt.utils.custom_op import register_custom_op @@ -80,16 +76,29 @@ def _qknorm_complex_rope_onepass_kernel( ) -def can_use_qknorm_complex_rope(x, weight, rope): - return ( - can_use_rmsnorm_preserve_reduction(x, weight) - and can_use_fused_complex_rope(x, rope) +def _validate_qknorm_complex_rope(x, weight, rope): + if not ( + x.is_cuda + and torch.version.hip is None + and x.dtype in (torch.float16, torch.bfloat16) + and x.ndim == 4 and x.shape[-1] == 128 - ) + and x.numel() > 0 + and x.is_contiguous() + and weight.device == x.device + and weight.dtype == x.dtype + and weight.shape == (128,) + and weight.is_contiguous() + and rope.dtype == torch.complex64 + and rope.device == x.device + and rope.shape == (x.shape[1], 64) + and rope.is_contiguous() + ): + raise RuntimeError("invalid input for qknorm_complex_rope") def _fake_qknorm_complex_rope(x, weight, rope, eps): - return torch.empty_like(x) + return torch.empty(x.shape, dtype=x.dtype, device=x.device) @register_custom_op( @@ -103,7 +112,8 @@ def qknorm_complex_rope( rope: torch.Tensor, eps: float, ) -> torch.Tensor: - assert can_use_qknorm_complex_rope(x, weight, rope) + x = x.contiguous() + _validate_qknorm_complex_rope(x, weight, rope) out = torch.empty_like(x) with torch.cuda.device(x.device): _qknorm_complex_rope_onepass_kernel[(triton.cdiv(x.numel() // 128, 4),)]( diff --git a/python/sglang/kernels/ops/diffusion/rope/rope_rotate_half_bitexact.py b/python/sglang/kernels/ops/diffusion/rope/rope_rotate_half_bitexact.py index a0d6af11c3..f574452ccf 100644 --- a/python/sglang/kernels/ops/diffusion/rope/rope_rotate_half_bitexact.py +++ b/python/sglang/kernels/ops/diffusion/rope/rope_rotate_half_bitexact.py @@ -83,32 +83,6 @@ def _rope_rotate_half_kernel( tl.store(out_ptr + toff, tail, mask=tmask) -def can_use_fused_rope_rotate_half( - x: torch.Tensor, - cos: torch.Tensor, - sin: torch.Tensor, -) -> bool: - if x.dtype is not torch.bfloat16 or not x.is_cuda: - return False - if x.dim() != 4 or not x.is_contiguous(): - return False - rows = x.shape[0] * x.shape[1] - rot = cos.shape[-1] - return ( - cos.dtype is torch.bfloat16 - and sin.dtype is torch.bfloat16 - and cos.is_cuda - and cos.device == x.device - and sin.device == x.device - and cos.shape == (rows, rot) - and sin.shape == (rows, rot) - and cos.is_contiguous() - and sin.is_contiguous() - and rot % 2 == 0 - and 0 < rot <= x.shape[-1] - ) - - def _fake_rope_rotate_half( x: torch.Tensor, cos: torch.Tensor, @@ -129,15 +103,36 @@ def fused_rope_rotate_half_bitexact( ) -> torch.Tensor: """Rotate-half RoPE over the leading ``cos.shape[-1]`` columns of ``x``. - ``x`` is ``(B, S, H, D)``; ``cos``/``sin`` are ``(B * S, rot_dim)`` rows. + ``x`` is ``(B, S, H, D)``; ``cos``/``sin`` are contiguous + ``(B * S, rot_dim)`` rows or ``(B, S, 1, rot_dim)`` broadcast tables. Bit-exact vs the eager chunk/neg/cat/mul/add chain. """ + if not ( + x.is_cuda and x.dtype is torch.bfloat16 and x.ndim == 4 and x.is_contiguous() + ): + raise RuntimeError("rotate-half RoPE expects contiguous BF16 CUDA [B, S, H, D]") batch, seq_len, heads, head_dim = x.shape + if cos.ndim not in (2, 4): + raise RuntimeError("cos must have shape [B * S, rot_dim] or [B, S, 1, rot_dim]") rot = cos.shape[-1] + if not (0 < rot <= head_dim and rot % 2 == 0): + raise RuntimeError("rot_dim must be positive, even and no larger than head_dim") + table_shape = (batch * seq_len, rot) if cos.ndim == 2 else (batch, seq_len, 1, rot) + device = x.device + for name, tensor in (("cos", cos), ("sin", sin)): + if not ( + tensor.dtype is torch.bfloat16 + and tensor.device == device + and tensor.shape == table_shape + and tensor.is_contiguous() + ): + raise RuntimeError( + f"{name} must be contiguous {table_shape} with x's dtype/device" + ) half = rot // 2 out = torch.empty_like(x) tail = head_dim - rot - with torch.cuda.device(x.device): + with torch.cuda.device(device): _rope_rotate_half_kernel[(batch * seq_len,)]( out, x, diff --git a/python/sglang/kernels/ops/diffusion/sites/ltx2_rmsnorm_modulate_site.py b/python/sglang/kernels/ops/diffusion/sites/ltx2_rmsnorm_modulate_site.py index 1bad9f14bf..980681d9f2 100644 --- a/python/sglang/kernels/ops/diffusion/sites/ltx2_rmsnorm_modulate_site.py +++ b/python/sglang/kernels/ops/diffusion/sites/ltx2_rmsnorm_modulate_site.py @@ -22,7 +22,6 @@ import torch from torch import nn from sglang.kernels.ops.diffusion.norm.rmsnorm_scale_shift_bitexact import ( - can_use_fused_rmsnorm_scale_shift, fused_rmsnorm_scale_shift_bitexact, ) from sglang.kernels.ops.diffusion.sites.quality_gate import QualityGatedFusion @@ -65,14 +64,6 @@ def _ones_weight(x: torch.Tensor) -> torch.Tensor: return w -def can_use_ltx2_rms_norm_modulate( - x: torch.Tensor, scale: torch.Tensor, shift: torch.Tensor -) -> bool: - if x.dtype is not torch.bfloat16 or not x.is_cuda: - return False - return can_use_fused_rmsnorm_scale_shift(x, _ones_weight(x), scale, shift) - - def fused_ltx2_rms_norm_modulate( x: torch.Tensor, scale: torch.Tensor, shift: torch.Tensor, eps: float ) -> torch.Tensor: diff --git a/python/sglang/multimodal_gen/runtime/models/decoders/ltx_2_5_diffusion_decoder.py b/python/sglang/multimodal_gen/runtime/models/decoders/ltx_2_5_diffusion_decoder.py index ac8d86f9f7..90b6dd2e00 100644 --- a/python/sglang/multimodal_gen/runtime/models/decoders/ltx_2_5_diffusion_decoder.py +++ b/python/sglang/multimodal_gen/runtime/models/decoders/ltx_2_5_diffusion_decoder.py @@ -19,7 +19,6 @@ from torch import nn from sglang.kernels.ops.diffusion import ( BitExactFusionGate, - can_use_ltx25_decoder_rope, fused_ltx25_decoder_rope, tensors_equal, ) @@ -32,8 +31,11 @@ from sglang.multimodal_gen.runtime.layers.visual_embedding import ( from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import ( LayerwiseOffloadableModuleMixin, ) +from sglang.multimodal_gen.runtime.platforms import current_platform from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger +_is_cuda = current_platform.is_cuda() + logger = init_logger(__name__) # Bounds the three hidden-width temporaries the SwiGLU holds live. Exact: the @@ -326,7 +328,9 @@ class LTX2VideoVaeRotaryPosEmbed3D(nn.Module): verified = _LTX25_DECODER_ROPE.verified if ( not _LTX25_DECODER_ROPE.disabled - and can_use_ltx25_decoder_rope(query, key, tables, self.rope_dim_split) + and _is_cuda + and query.is_cuda + and query.dtype is torch.bfloat16 and (verified or _LTX25_DECODER_ROPE.can_attempt_once()) ): dim_t, dim_h, _ = self.rope_dim_split diff --git a/python/sglang/multimodal_gen/runtime/models/dits/ernie_image.py b/python/sglang/multimodal_gen/runtime/models/dits/ernie_image.py index be94f0f7ec..856bed68d8 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/ernie_image.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/ernie_image.py @@ -25,8 +25,6 @@ from sglang.kernels.ops.activation.activation import ( from sglang.kernels.ops.diffusion import ( BitExactFusionGate, can_use_fused_rmsnorm_scale_shift, - can_use_fused_rope_rotate_half, - can_use_fused_scale_residual_rmsnorm_scale_shift, flashinfer_rmsnorm_diagnostic_hint, fused_rmsnorm_scale_shift_bitexact, fused_rope_rotate_half_bitexact, @@ -92,8 +90,9 @@ def _ernie_norm_scale_shift( verified = _ERNIE_NORM.verified if ( not _ERNIE_NORM.disabled + and x.is_cuda and norm.variance_size_override is None - and can_use_fused_rmsnorm_scale_shift(x, norm.weight, scale, shift) + and can_use_fused_rmsnorm_scale_shift(x.dtype, x.shape[-1]) and (verified or _ERNIE_NORM.can_attempt_once()) ): try: @@ -136,10 +135,9 @@ def _ernie_gated_norm_scale_shift( verified = _ERNIE_GATED_NORM.verified if ( not _ERNIE_GATED_NORM.disabled + and residual.is_cuda and norm.variance_size_override is None - and can_use_fused_scale_residual_rmsnorm_scale_shift( - residual, update, gate, norm.weight, scale, shift - ) + and can_use_fused_rmsnorm_scale_shift(residual.dtype, residual.shape[-1]) and (verified or _ERNIE_GATED_NORM.can_attempt_once()) ): try: @@ -461,7 +459,8 @@ def _ernie_rope( verified = _ERNIE_ROPE.verified if ( not _ERNIE_ROPE.disabled - and can_use_fused_rope_rotate_half(x, cos_, sin_) + and x.is_cuda + and x.dtype is torch.bfloat16 and (verified or _ERNIE_ROPE.can_attempt_once()) ): try: diff --git a/python/sglang/multimodal_gen/runtime/models/dits/flux.py b/python/sglang/multimodal_gen/runtime/models/dits/flux.py index 43b5b6cd9e..da5b71fd62 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/flux.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/flux.py @@ -122,8 +122,9 @@ def _flux_fused_ln_modulate( """ if ( _FLUX_LN_MOD.disabled + or not x.is_cuda or not is_plain_layer_norm(norm, x.shape[-1]) - or not can_use_fused_layernorm_modulate(x, scale, shift) + or not can_use_fused_layernorm_modulate(x.dtype, x.shape[-1]) ): return None sig = ( diff --git a/python/sglang/multimodal_gen/runtime/models/dits/flux3.py b/python/sglang/multimodal_gen/runtime/models/dits/flux3.py index 16a7bcd36b..ce8e4a08d0 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/flux3.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/flux3.py @@ -153,7 +153,7 @@ def _norm_modulate( _LN_MODULATE.disabled or not _eager_fast_path_allowed(x) or not is_plain_layer_norm(norm, x.shape[-1]) - or not can_use_fused_layernorm_modulate(x, scale_row, shift_row) + or not can_use_fused_layernorm_modulate(x.dtype, x.shape[-1]) ): return (1 + scale) * norm(x) + shift sig = (x.device, x.shape[0], x.shape[-1], norm.eps) diff --git a/python/sglang/multimodal_gen/runtime/models/dits/flux_2.py b/python/sglang/multimodal_gen/runtime/models/dits/flux_2.py index d5854576cc..7fc9e17801 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/flux_2.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/flux_2.py @@ -208,9 +208,10 @@ def _try_flux2_norm_modulate_fp8( shift_row = shift.squeeze(1) if shift.dim() == 3 and shift.shape[1] == 1 else shift if ( _FLUX2_LN_FP8.disabled + or not x.is_cuda or x.shape[-1] != 6144 or not is_plain_layer_norm(norm, x.shape[-1]) - or not can_use_fused_layernorm_modulate(x, scale_row, shift_row) + or not can_use_fused_layernorm_modulate(x.dtype, x.shape[-1]) ): return None @@ -373,8 +374,9 @@ def _flux2_norm_modulate( shift_row = shift.squeeze(1) if shift.dim() == 3 and shift.shape[1] == 1 else shift if ( _FLUX2_LN_MOD.disabled + or not x.is_cuda or not is_plain_layer_norm(norm, x.shape[-1]) - or not can_use_fused_layernorm_modulate(x, scale_row, shift_row) + or not can_use_fused_layernorm_modulate(x.dtype, x.shape[-1]) ): return norm(x) * (1 + scale) + shift diff --git a/python/sglang/multimodal_gen/runtime/models/dits/glm_image.py b/python/sglang/multimodal_gen/runtime/models/dits/glm_image.py index 9e49f5eb8f..5fd03e68af 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/glm_image.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/glm_image.py @@ -103,9 +103,10 @@ def _glm_ln_modulate( if ( not _GLM_LN_MOD.disabled and _is_cuda + and x.is_cuda and dtype is x.dtype and is_plain_layer_norm(norm, x.shape[-1]) - and can_use_fused_layernorm_modulate(x, scale, shift) + and can_use_fused_layernorm_modulate(x.dtype, x.shape[-1]) and (verified or _GLM_LN_MOD.can_attempt_once()) ): try: @@ -144,12 +145,13 @@ def _glm_qk_layernorm( if ( not _GLM_QK_LN.disabled and _is_cuda + and query.is_cuda and dtype is query.dtype and dtype is key.dtype and is_plain_layer_norm(norm_q, query.shape[-1]) and is_plain_layer_norm(norm_k, key.shape[-1]) and norm_q.eps == norm_k.eps - and can_use_fused_qk_head_layernorm(query, key) + and can_use_fused_qk_head_layernorm(query.dtype, query.shape[-1]) and (verified or _GLM_QK_LN.can_attempt_once()) ): try: diff --git a/python/sglang/multimodal_gen/runtime/models/dits/helios.py b/python/sglang/multimodal_gen/runtime/models/dits/helios.py index 1f72217545..02958a0c45 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/helios.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/helios.py @@ -17,7 +17,6 @@ import torch.nn as nn import torch.nn.functional as F from sglang.kernels.ops.diffusion import ( - can_use_helios_qk_rope, fused_inplace_helios_qk_rope, mark_helios_gated_residual_site, try_helios_gated_residual, @@ -58,8 +57,11 @@ from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload im LayerwiseOffloadableModuleMixin, ) from sglang.multimodal_gen.runtime.models.dits.base import CachableDiT +from sglang.multimodal_gen.runtime.platforms import current_platform from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger +_is_cuda = current_platform.is_cuda() + logger = init_logger(__name__) @@ -297,12 +299,8 @@ class HeliosSelfAttention(nn.Module): k: torch.Tensor, rotary_emb: torch.Tensor, ) -> tuple[torch.Tensor, torch.Tensor]: - if not self.tp_rmsnorm and can_use_helios_qk_rope(q, k, rotary_emb): - fused_inplace_helios_qk_rope( - q.view(-1, q.shape[-2], q.shape[-1]), - k.view(-1, k.shape[-2], k.shape[-1]), - rotary_emb.view(-1, rotary_emb.shape[-1]), - ) + if not self.tp_rmsnorm and _is_cuda and q.is_cuda: + fused_inplace_helios_qk_rope(q, k, rotary_emb) return q, k return ( apply_rotary_emb_transposed(q, rotary_emb), diff --git a/python/sglang/multimodal_gen/runtime/models/dits/ideogram.py b/python/sglang/multimodal_gen/runtime/models/dits/ideogram.py index 691283302d..c9e8332751 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/ideogram.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/ideogram.py @@ -9,7 +9,6 @@ import torch.nn.functional as F from sglang.kernels.ops.diffusion import ( BitExactFusionGate, - can_use_fused_silu_mul, fused_gate_rmsnorm_active, fused_rmsnorm_scale, fused_rmsnorm_tanh_residual, @@ -54,9 +53,14 @@ from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload im LayerwiseOffloadableModuleMixin, ) from sglang.multimodal_gen.runtime.models.dits.base import BaseDiT -from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum +from sglang.multimodal_gen.runtime.platforms import ( + AttentionBackendEnum, + current_platform, +) from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger +_is_cuda = current_platform.is_cuda() + logger = init_logger(__name__) OUTPUT_IMAGE_INDICATOR = 2 @@ -67,35 +71,6 @@ _IDEOGRAM_SWIGLU = BitExactFusionGate("Ideogram fused SiLU-mul") _IDEOGRAM_ZERO_SHIFTS: dict[tuple[torch.device, torch.dtype, int], torch.Tensor] = {} -def _can_use_fused_rope( - q: torch.Tensor, - k: torch.Tensor, - cos: torch.Tensor, - sin: torch.Tensor, -) -> bool: - # cos/sin are full-span (B, S, 1, D) rows broadcast over heads. - expected = (q.shape[0], q.shape[1], 1, q.shape[-1]) - return ( - q.dtype is torch.bfloat16 - and k.dtype is torch.bfloat16 - and q.is_cuda - and q.dim() == 4 - and q.is_contiguous() - and k.shape == q.shape - and k.is_contiguous() - and k.device == q.device - and cos.dtype is torch.bfloat16 - and sin.dtype is torch.bfloat16 - and cos.device == q.device - and sin.device == q.device - and cos.shape == expected - and sin.shape == expected - and cos.is_contiguous() - and sin.is_contiguous() - and q.shape[-1] % 2 == 0 - ) - - def _ideogram_rope( q: torch.Tensor, k: torch.Tensor, @@ -113,14 +88,15 @@ def _ideogram_rope( verified = _IDEOGRAM_ROPE.verified if ( not _IDEOGRAM_ROPE.disabled - and _can_use_fused_rope(q, k, cos, sin) + and q.is_cuda + and q.dtype is torch.bfloat16 + and q.is_contiguous() + and k.is_contiguous() and (verified or _IDEOGRAM_ROPE.can_attempt_once()) ): try: - cos_rows = cos.reshape(-1, cos.shape[-1]) - sin_rows = sin.reshape(-1, sin.shape[-1]) - q_fused = fused_rope_rotate_half_bitexact(q, cos_rows, sin_rows) - k_fused = fused_rope_rotate_half_bitexact(k, cos_rows, sin_rows) + q_fused = fused_rope_rotate_half_bitexact(q, cos, sin) + k_fused = fused_rope_rotate_half_bitexact(k, cos, sin) except Exception as exc: _IDEOGRAM_ROPE.on_exception(exc, logger=logger) else: @@ -144,7 +120,9 @@ def _ideogram_swiglu(a: torch.Tensor, b: torch.Tensor) -> torch.Tensor: verified = _IDEOGRAM_SWIGLU.verified if ( not _IDEOGRAM_SWIGLU.disabled - and can_use_fused_silu_mul(a, b) + and _is_cuda + and a.is_cuda + and a.dtype is torch.bfloat16 and (verified or _IDEOGRAM_SWIGLU.can_attempt_once()) ): try: diff --git a/python/sglang/multimodal_gen/runtime/models/dits/joy_image.py b/python/sglang/multimodal_gen/runtime/models/dits/joy_image.py index 87f0906332..c1e91c5db3 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/joy_image.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/joy_image.py @@ -177,7 +177,9 @@ def _joy_joint_qkv(*inputs: torch.Tensor) -> tuple[torch.Tensor, ...]: or not inputs[0].is_cuda or torch.version.hip or torch.compiler.is_compiling() - or not diffusion_kernels.can_use_joint_qkv_cat(*inputs) + or inputs[0].dtype not in (torch.float16, torch.bfloat16) + or inputs[0].shape[-2] * inputs[0].shape[-1] > 8192 + or any(x.requires_grad for x in inputs) ): return reference() sig = (inputs[0].device, inputs[0].dtype) + tuple( diff --git a/python/sglang/multimodal_gen/runtime/models/dits/longcat_image.py b/python/sglang/multimodal_gen/runtime/models/dits/longcat_image.py index 559d7f2e92..ac6c2e9dbf 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/longcat_image.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/longcat_image.py @@ -58,8 +58,11 @@ from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload im LayerwiseOffloadableModuleMixin, ) from sglang.multimodal_gen.runtime.models.dits.base import BaseDiT +from sglang.multimodal_gen.runtime.platforms import current_platform from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger +_is_cuda = current_platform.is_cuda() + logger = init_logger(__name__) _LONGCAT_QKNORM_ROPE = BitExactFusionGate("LongCat fused QKNorm+RoPE") @@ -74,7 +77,11 @@ def _longcat_gelu_cat( type(activation) is nn.GELU and activation.approximate == "tanh" and _LONGCAT_GELU_CAT.can_attempt_once() - and diffusion_ops.can_use_fused_gelu_tanh_cat(attn, mlp) + and _is_cuda + and attn.is_cuda + and attn.dtype is torch.bfloat16 + and not torch.is_grad_enabled() + and not torch.compiler.is_compiling() ): try: output = diffusion_ops.fused_gelu_tanh_cat(attn, mlp) @@ -104,7 +111,7 @@ def _longcat_norm_modulate( not _LONGCAT_LN_MOD.disabled and x.is_cuda and diffusion_ops.is_plain_layer_norm(norm, x.shape[-1]) - and diffusion_ops.can_use_fused_layernorm_modulate(x, scale, shift) + and diffusion_ops.can_use_fused_layernorm_modulate(x.dtype, x.shape[-1]) ): sig = ( x.shape, diff --git a/python/sglang/multimodal_gen/runtime/models/dits/ltx_2.py b/python/sglang/multimodal_gen/runtime/models/dits/ltx_2.py index c2bca49618..3e39022a8e 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/ltx_2.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/ltx_2.py @@ -12,9 +12,9 @@ import torch.nn.functional as F from sglang.kernels.ops.diffusion import ( BitExactFusionGate, + can_use_fused_rmsnorm_scale_shift, can_use_linear_gelu, can_use_ltx2_qknorm_split_rope_cuda, - can_use_ltx2_rms_norm_modulate, can_use_modulate_scale_shift_cuda, fused_gelu_active, fused_linear_gelu_tanh, @@ -208,8 +208,10 @@ def _ltx2_rms_norm_modulate( default). The fused kernel is not bit-exact (<=1 bf16 ULP) so it is gated on the request-scoped mount rather than a runtime self-check. """ - if ltx2_rms_norm_modulate_active(block) and can_use_ltx2_rms_norm_modulate( - x, scale, shift + if ( + ltx2_rms_norm_modulate_active(block) + and x.is_cuda + and can_use_fused_rmsnorm_scale_shift(x.dtype, x.shape[-1]) ): return fused_ltx2_rms_norm_modulate(x, scale, shift, eps) normed = rms_norm(x, eps) diff --git a/python/sglang/multimodal_gen/runtime/models/dits/minimax_h3_vdn.py b/python/sglang/multimodal_gen/runtime/models/dits/minimax_h3_vdn.py index 4c0accace6..9b6d5f8f60 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/minimax_h3_vdn.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/minimax_h3_vdn.py @@ -520,7 +520,7 @@ def delta_factor_apply( B32.contiguous(), alpha.float().contiguous(), ) - if can_use_vdn_delta_factors(A32, B32, alpha32): + if can_use_vdn_delta_factors(A32.device, A32.dtype, A32.shape[-1]): return vdn_delta_factors(A32, B32, alpha32) chol = torch.linalg.cholesky(A32 + eye) # (I+A)^-1 = L^-T L^-1: a batched trsm at 128x128 is far slower than the GEMM diff --git a/python/sglang/multimodal_gen/runtime/models/dits/qwen_image.py b/python/sglang/multimodal_gen/runtime/models/dits/qwen_image.py index d1bf04f3dd..7b7ca0cc7c 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/qwen_image.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/qwen_image.py @@ -133,8 +133,11 @@ def _qwen_norm_out( scale, shift = torch.chunk(emb, 2, dim=1) if ( _QWEN_NORM_OUT.disabled + or not hidden_states.is_cuda or not is_plain_layer_norm(norm_out.norm, hidden_states.shape[-1]) - or not can_use_fused_layernorm_modulate(hidden_states, scale, shift) + or not can_use_fused_layernorm_modulate( + hidden_states.dtype, hidden_states.shape[-1] + ) ): return ( norm_out.norm(hidden_states) * (1 + scale)[:, None, :] + shift[:, None, :] diff --git a/python/sglang/multimodal_gen/runtime/models/dits/qwen_image21.py b/python/sglang/multimodal_gen/runtime/models/dits/qwen_image21.py index 228500928f..1241ec64fd 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/qwen_image21.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/qwen_image21.py @@ -8,12 +8,8 @@ from torch import nn from sglang.kernels.ops.diffusion import ( BitExactFusionGate, - can_use_fused_complex_rope, can_use_fused_layernorm_modulate, - can_use_fused_silu_mul, can_use_qknorm_complex_rope_cuda, - can_use_qknorm_complex_rope_pack, - can_use_rmsnorm_preserve_reduction, fused_complex_rope, fused_layernorm_modulate, fused_silu_mul_bitexact, @@ -24,11 +20,9 @@ from sglang.kernels.ops.diffusion import ( tensors_equal, ) from sglang.kernels.ops.diffusion.rope.qknorm_complex_rope_kv_triton import ( - can_use_qknorm_complex_rope_kv, qknorm_complex_rope_kv, ) from sglang.kernels.ops.diffusion.rope.qknorm_complex_rope_triton import ( - can_use_qknorm_complex_rope, qknorm_complex_rope, ) from sglang.multimodal_gen.runtime.distributed import ( @@ -51,10 +45,15 @@ from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload im LayerwiseOffloadableModuleMixin, ) from sglang.multimodal_gen.runtime.models.dits.base import CachableDiT -from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum +from sglang.multimodal_gen.runtime.platforms import ( + AttentionBackendEnum, + current_platform, +) from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger from sglang.srt.layers.layernorm import RMSNorm +_is_cuda = current_platform.is_cuda() + logger = init_logger(__name__) _ROPE_FUSION = BitExactFusionGate("Qwen-Image 2.1 complex RoPE") _SILU_MUL_FUSION = BitExactFusionGate("Qwen-Image 2.1 SiLU-mul") @@ -130,7 +129,7 @@ def build_layout(image_slots, image_shapes, axes_dims, device): def apply_rope(x, rope): fused = None - if can_use_fused_complex_rope(x, rope) and _ROPE_FUSION.can_attempt_once(): + if _is_cuda and x.is_cuda and _ROPE_FUSION.can_attempt_once(): fused = fused_complex_rope(x, rope) if _ROPE_FUSION.verified: return fused @@ -144,7 +143,9 @@ def apply_rope(x, rope): def apply_qk_norm(x, norm): fused = None if ( - can_use_rmsnorm_preserve_reduction(x, norm.weight) + _is_cuda + and x.is_cuda + and x.dtype in (torch.float16, torch.bfloat16) and _QK_NORM_FUSION.can_attempt_once() ): fused = rmsnorm_preserve_reduction(x, norm.weight, norm.variance_epsilon) @@ -159,22 +160,28 @@ def apply_qk_norm(x, norm): def apply_qk_norm_rope(x, norm, rope): fused = None if ( - can_use_qknorm_complex_rope_cuda(x, norm.weight, rope) + _is_cuda + and x.is_cuda + and not torch.compiler.is_compiling() + and can_use_qknorm_complex_rope_cuda(x.dtype, x.shape[-1]) and _QK_ROPE_CUDA_FUSION.can_attempt_once() ): fused = qknorm_complex_rope_cuda(x, norm.weight, rope, norm.variance_epsilon) if _QK_ROPE_CUDA_FUSION.verified: return fused - out = apply_rope(apply_qk_norm(x, norm), rope) + out = apply_rope(norm(x), rope) return _QK_ROPE_CUDA_FUSION.accept_or_fallback(fused, out, logger=logger) if ( - can_use_qknorm_complex_rope(x, norm.weight, rope) + _is_cuda + and x.is_cuda + and x.dtype in (torch.float16, torch.bfloat16) + and x.shape[-1] == 128 and _QK_ROPE_FUSION.can_attempt_once() ): fused = qknorm_complex_rope(x, norm.weight, rope, norm.variance_epsilon) if _QK_ROPE_FUSION.verified: return fused - out = apply_rope(apply_qk_norm(x, norm), rope) + out = apply_rope(norm(x), rope) if fused is not None: return _QK_ROPE_FUSION.accept_or_fallback(fused, out, logger=logger) return out @@ -218,7 +225,8 @@ def cat_outputs(outputs, dim=0): def apply_modulation(x, norm, scale): fused = None if ( - can_use_fused_layernorm_modulate(x, scale.squeeze(1), None) + x.is_cuda + and can_use_fused_layernorm_modulate(x.dtype, x.shape[-1]) and _MODULATION_FUSION.can_attempt_once() ): fused = fused_layernorm_modulate(x, scale.squeeze(1), None, norm.eps) @@ -305,7 +313,11 @@ class QwenImage21FeedForward(nn.Module): def forward(self, x): gate, value = self.gate_layer(x)[0], self.proj(x)[0] fused = None - if can_use_fused_silu_mul(gate, value) and _SILU_MUL_FUSION.can_attempt_once(): + if ( + _is_cuda + and gate.dtype is torch.bfloat16 + and _SILU_MUL_FUSION.can_attempt_once() + ): fused = fused_silu_mul_bitexact(gate, value) if _SILU_MUL_FUSION.verified: return self.out(fused)[0] @@ -443,9 +455,16 @@ class QwenImage21Attention(nn.Module): With ``k_out``/``v_out`` the projection already wrote the raw K/V into rows ``[P:]`` (``k``/``v`` are those views), so K is normalized in place and V is not copied. Returns ``(q, k_out, v_out)`` or ``None`` when the - layout is unsupported. The first call also runs the eager chain and + dtype/head dimension is unsupported. The first call also runs the eager chain and keeps its result on a mismatch. """ + if not ( + _is_cuda + and q.is_cuda + and not torch.compiler.is_compiling() + and can_use_qknorm_complex_rope_cuda(q.dtype, self.head_dim) + ): + return None eps = self.norm_k.variance_epsilon if self.norm_q.variance_epsilon != eps: return None @@ -456,24 +475,11 @@ class QwenImage21Attention(nn.Module): k_src, v_src = k, v else: k_src, v_src = None, None - if not can_use_qknorm_complex_rope_pack( - q, - k_out, - v_out, - self.norm_q.weight, - self.norm_k.weight, - rope, - kp, - vp, - k_src, - v_src, - ): - return None reference = None if not _KV_PACK_CUDA_FUSION.verified: reference = ( - apply_rope(apply_qk_norm(q, self.norm_q), rope), - torch.cat([kp, apply_rope(apply_qk_norm(k, self.norm_k), rope)], 1), + apply_rope(self.norm_q(q), rope), + torch.cat([kp, apply_rope(self.norm_k(k), rope)], 1), torch.cat([vp, v], 1), ) qknorm_complex_rope_pack_( @@ -612,7 +618,10 @@ class QwenImage21Attention(nn.Module): packed = None if ( get_sp_world_size() == 1 - and can_use_qknorm_complex_rope_kv(k, self.norm_k.weight, rope, v, kp, vp) + and _is_cuda + and k.is_cuda + and k.dtype in (torch.float16, torch.bfloat16) + and self.head_dim == 128 and _KV_ROPE_FUSION.can_attempt_once() ): packed = qknorm_complex_rope_kv( @@ -620,7 +629,7 @@ class QwenImage21Attention(nn.Module): ) if not _KV_ROPE_FUSION.verified: reference = ( - torch.cat([kp, apply_rope(apply_qk_norm(k, self.norm_k), rope)], 1), + torch.cat([kp, apply_rope(self.norm_k(k), rope)], 1), torch.cat([vp, v], 1), ) packed = _KV_ROPE_FUSION.accept_or_fallback( diff --git a/python/sglang/multimodal_gen/runtime/models/dits/sana.py b/python/sglang/multimodal_gen/runtime/models/dits/sana.py index 73df6333a2..b3768d35cb 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/sana.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/sana.py @@ -7,8 +7,6 @@ from diffusers.models.embeddings import PixArtAlphaTextProjection, TimestepEmbed from sglang.kernels.ops.diffusion import ( BitExactFusionGate, - can_use_fused_bias_glu, - can_use_fused_bias_silu, can_use_fused_layernorm_modulate, fused_bias_glu, fused_bias_silu, @@ -106,7 +104,7 @@ def sana_ln_modulate( and is_plain_layer_norm(norm, x.shape[-1]) ): x_c = x.contiguous() - if not can_use_fused_layernorm_modulate(x_c, scale[:, 0], shift[:, 0]): + if not can_use_fused_layernorm_modulate(x_c.dtype, x_c.shape[-1]): return _eager_ln_modulate(norm, x, scale, shift) try: out = fused_layernorm_modulate_raw(x_c, scale[:, 0], shift[:, 0], norm.eps) @@ -185,7 +183,7 @@ def sana_conv_bias_silu( return F.silu(_mps_safe_conv2d(conv, x)) raw = _conv2d_without_bias(conv, x) - if not can_use_fused_bias_silu(raw, conv.bias): + if raw.dtype is not torch.bfloat16: return F.silu(raw + conv.bias[None, :, None, None]) verified = _SANA_CONV_SILU.verified if not verified and not _SANA_CONV_SILU.can_attempt_once(): @@ -217,10 +215,8 @@ def sana_conv_bias_glu( return hidden_states * F.silu(gate) raw = _conv2d_without_bias(conv, x) - if not can_use_fused_bias_glu(raw, conv.bias): - hidden_states, gate = torch.chunk( - raw + conv.bias[None, :, None, None], 2, dim=1 - ) + if raw.dtype is not torch.bfloat16: + hidden_states, gate = (raw + conv.bias[None, :, None, None]).chunk(2, dim=1) return hidden_states * F.silu(gate) verified = _SANA_CONV_GLU.verified if not verified and not _SANA_CONV_GLU.can_attempt_once(): diff --git a/python/sglang/multimodal_gen/runtime/models/dits/sana_video.py b/python/sglang/multimodal_gen/runtime/models/dits/sana_video.py index a2b11e2be7..b347799381 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/sana_video.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/sana_video.py @@ -11,7 +11,6 @@ from diffusers.models.embeddings import PixArtAlphaTextProjection from sglang.kernels.ops.diffusion import ( BitExactFusionGate, - can_use_interleaved_rope_fp64, fused_interleaved_rope_fp64, mark_sana_video_linear_attention_site, tensors_equal, @@ -34,8 +33,11 @@ from sglang.multimodal_gen.runtime.models.dits.sana import ( sana_ln_modulate, sana_residual_gate_add, ) +from sglang.multimodal_gen.runtime.platforms import current_platform from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger +_is_cuda = current_platform.is_cuda() + logger = init_logger(__name__) _SANA_VIDEO_ROPE = BitExactFusionGate("SANA-Video fused RoPE") @@ -66,7 +68,9 @@ def apply_interleaved_rotary_emb_pair( verified = _SANA_VIDEO_ROPE.verified if ( not _SANA_VIDEO_ROPE.disabled - and can_use_interleaved_rope_fp64(query, key, freqs_cos, freqs_sin) + and _is_cuda + and query.is_cuda + and query.dtype is torch.bfloat16 and (verified or _SANA_VIDEO_ROPE.can_attempt_once()) ): try: diff --git a/python/sglang/multimodal_gen/runtime/models/dits/wanvideo.py b/python/sglang/multimodal_gen/runtime/models/dits/wanvideo.py index e3d63c5ebe..ec83bfc922 100755 --- a/python/sglang/multimodal_gen/runtime/models/dits/wanvideo.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/wanvideo.py @@ -11,7 +11,6 @@ import torch.nn as nn from sglang.kernels.ops.diffusion import ( BitExactFusionGate, - can_use_fused_temb_table_slices, can_use_linear_gelu, fused_gelu_active, fused_linear_gelu_tanh, @@ -410,7 +409,8 @@ def _wan_temb_table_slices( verified = _WAN_TEMB_SLICES.verified if ( not _WAN_TEMB_SLICES.disabled - and can_use_fused_temb_table_slices(table, temb) + and _is_cuda + and temb.is_cuda and (verified or _WAN_TEMB_SLICES.can_attempt_once()) ): try: diff --git a/python/sglang/multimodal_gen/runtime/utils/model_overlay.py b/python/sglang/multimodal_gen/runtime/utils/model_overlay.py index 9ea9c6365c..18515523c9 100644 --- a/python/sglang/multimodal_gen/runtime/utils/model_overlay.py +++ b/python/sglang/multimodal_gen/runtime/utils/model_overlay.py @@ -576,17 +576,20 @@ def materialize_overlay_model( overlay_repo_id = str(overlay_spec["overlay_repo_id"]) overlay_revision = str(overlay_spec.get("overlay_revision", "main")) overlay_fingerprint = _compute_overlay_fingerprint(overlay_dir) + source_revision = source_revision or _overlay_source_revision( + overlay_spec, manifest + ) + cache_identity = { + "source_model_id": source_model_id, + "overlay_repo_id": overlay_repo_id, + "overlay_revision": overlay_revision, + "materializer_version": materializer_version, + "overlay_fingerprint": overlay_fingerprint, + } + if source_revision is not None: + cache_identity["source_revision"] = source_revision cache_key = hashlib.sha256( - json.dumps( - { - "source_model_id": source_model_id, - "overlay_repo_id": overlay_repo_id, - "overlay_revision": overlay_revision, - "materializer_version": materializer_version, - "overlay_fingerprint": overlay_fingerprint, - }, - sort_keys=True, - ).encode("utf-8") + json.dumps(cache_identity, sort_keys=True).encode("utf-8") ).hexdigest()[:16] cache_root = os.path.join(get_diffusion_cache_root(), "materialized_models") _ensure_dir(cache_root) diff --git a/python/sglang/multimodal_gen/test/runner/diffusion_suite_runner.py b/python/sglang/multimodal_gen/test/runner/diffusion_suite_runner.py index 6348036292..decf4d642b 100644 --- a/python/sglang/multimodal_gen/test/runner/diffusion_suite_runner.py +++ b/python/sglang/multimodal_gen/test/runner/diffusion_suite_runner.py @@ -449,8 +449,19 @@ def _run_file_suite(args, target_dir: Path) -> int: print(f"No valid test files found for suite '{args.suite}'.") return 1 if args.suite in STRICT_SUITES else 0 + my_files = partition_items_by_index( + suite_files_abs, args.partition_id, args.total_partitions + ) + print( + f"Suite: {args.suite} | Partition: {args.partition_id}/{args.total_partitions} " + f"| Files: {len(my_files)}/{len(suite_files_abs)}", + flush=True, + ) + if not my_files: + return 0 + exit_code, _, _ = run_pytest( - suite_files_abs, + my_files, filter_expr=args.filter, junit_xml_path=None, ) diff --git a/python/sglang/multimodal_gen/test/unit/test_component_residency.py b/python/sglang/multimodal_gen/test/unit/test_component_residency.py index 903b89a2a1..af70919a48 100644 --- a/python/sglang/multimodal_gen/test/unit/test_component_residency.py +++ b/python/sglang/multimodal_gen/test/unit/test_component_residency.py @@ -356,10 +356,13 @@ class _Stage: ["--component-residency", "all=component-offload"], "component-offload", ), - ( + pytest.param( "text_encoder_2", ["--component-residency", "text_encoder=snapshot-offload"], "snapshot-offload", + marks=pytest.mark.skipif( + not current_platform.is_cuda(), reason="snapshot-offload requires CUDA" + ), ), ("text_encoder_2", ["--text-encoder-cpu-offload", "true"], "component-offload"), ("text_encoder_2", ["--text-encoder-cpu-offload", "false"], "resident"), diff --git a/python/sglang/multimodal_gen/test/unit/test_load_inclusive_e2e.py b/python/sglang/multimodal_gen/test/unit/test_load_inclusive_e2e.py index 3d2d3b00a7..839eb3be9d 100644 --- a/python/sglang/multimodal_gen/test/unit/test_load_inclusive_e2e.py +++ b/python/sglang/multimodal_gen/test/unit/test_load_inclusive_e2e.py @@ -24,6 +24,7 @@ from sglang.multimodal_gen.test.server.testcase_configs import ( @pytest.fixture def validator(monkeypatch): monkeypatch.setenv("SGLANG_GEN_BASELINE", "0") + monkeypatch.setattr(utils.current_platform, "is_hip", lambda: False) scenario = ScenarioConfig.from_dict( { "stages_ms": {}, diff --git a/python/sglang/multimodal_gen/test/unit/test_model_overlay.py b/python/sglang/multimodal_gen/test/unit/test_model_overlay.py index bc3978dce2..6f35e3e16a 100644 --- a/python/sglang/multimodal_gen/test/unit/test_model_overlay.py +++ b/python/sglang/multimodal_gen/test/unit/test_model_overlay.py @@ -1,6 +1,11 @@ """Unit tests for diffusion model-overlay cache paths and materialized trees.""" +import json import os +from pathlib import Path + +import pytest +from filelock import FileLock from sglang.multimodal_gen.runtime.utils import model_overlay from sglang.multimodal_gen.runtime.utils.model_overlay import ( @@ -65,6 +70,80 @@ def test_large_files_are_still_shared_with_the_cache(tmp_path, monkeypatch): assert os.path.samefile(out / "model.safetensors", weights) +@pytest.mark.parametrize("repair_cache", [False, True]) +def test_overlay_uses_pinned_source_weights(tmp_path, monkeypatch, repair_cache): + overlay = tmp_path / "overlay" + (overlay / "_overlay").mkdir(parents=True) + (overlay / "_overlay" / "overlay_manifest.json").write_text( + json.dumps( + { + "required_source_files": ["model.safetensors"], + "file_mappings": [ + {"src": "model.safetensors", "dst": "transformer/model.safetensors"} + ], + } + ) + ) + (overlay / "model_index.json").write_text( + json.dumps( + { + "_class_name": "TestPipeline", + "_diffusers_version": "0.35.0", + "transformer": ["diffusers", "Transformer"], + } + ) + ) + for revision in ("main", "original", "newer"): + source = tmp_path / revision + source.mkdir() + if revision != "original" or not repair_cache: + (source / "model.safetensors").write_text(revision) + + spec = {"overlay_repo_id": "test/overlay", "source_revision": "original"} + monkeypatch.setattr( + model_overlay, "resolve_model_overlay_target", lambda _: ("test/model", spec) + ) + monkeypatch.setattr( + model_overlay, "download_overlay_metadata", lambda *a, **kw: str(overlay) + ) + monkeypatch.setattr( + model_overlay, "get_diffusion_cache_root", lambda: str(tmp_path / "cache") + ) + monkeypatch.setattr( + model_overlay, "get_lock", lambda _: FileLock(tmp_path / "overlay.lock") + ) + + def cached_snapshot(*args, revision=None, **kwargs): + return str(tmp_path / (revision or "main")) + + def download_snapshot(*args, revision=None, **kwargs): + source = Path(cached_snapshot(revision=revision)) + (source / "model.safetensors").write_text(revision or "main") + return str(source) + + def resolve(): + return model_overlay.maybe_resolve_overlay_model_path( + "test/model", + local_dir=None, + download=True, + allow_patterns=None, + snapshot_download_fn=download_snapshot, + hf_hub_download_fn=None, + verify_diffusers_model_complete_fn=lambda path: os.path.isfile( + os.path.join(path, "model_index.json") + ), + base_model_download_fn=cached_snapshot, + ) + + original = Path(resolve()) + assert (original / "transformer/model.safetensors").read_text() == "original" + # A new source revision must not reuse weights materialized from the old one. + spec["source_revision"] = "newer" + newer = Path(resolve()) + assert (newer / "transformer/model.safetensors").read_text() == "newer" + assert original != newer + + def test_registry_source_revision_survives_upstream_removing_required_file( tmp_path, monkeypatch ): diff --git a/python/sglang/multimodal_gen/test/unit/test_qwen_image21_cuda.py b/python/sglang/multimodal_gen/test/unit/test_qwen_image21_cuda.py index b73d0a742b..f36682f4dc 100644 --- a/python/sglang/multimodal_gen/test/unit/test_qwen_image21_cuda.py +++ b/python/sglang/multimodal_gen/test/unit/test_qwen_image21_cuda.py @@ -38,6 +38,7 @@ from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.qwen_image21 import ( QwenImage21DenoisingStage, ) +from sglang.multimodal_gen.runtime.platforms import current_platform from sglang.multimodal_gen.runtime.server_args import ( ServerArgs, set_global_server_args, @@ -46,7 +47,10 @@ from sglang.multimodal_gen.test.single_test_file.component_accuracy.utils import ensure_distributed_env_defaults, ) -pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") +pytestmark = pytest.mark.skipif( + not (torch.cuda.is_available() and current_platform.is_cuda()), + reason="requires NVIDIA CUDA fusions", +) @pytest.fixture(scope="module") diff --git a/python/sglang/multimodal_gen/test/unit/test_runner_perf_baselines.py b/python/sglang/multimodal_gen/test/unit/test_runner_perf_baselines.py index 8cc9dd59ed..9357209517 100644 --- a/python/sglang/multimodal_gen/test/unit/test_runner_perf_baselines.py +++ b/python/sglang/multimodal_gen/test/unit/test_runner_perf_baselines.py @@ -2,6 +2,7 @@ from dataclasses import replace import pytest +from sglang.multimodal_gen.runtime.platforms import current_platform from sglang.multimodal_gen.test.server.test_server_utils import PerformanceValidator from sglang.multimodal_gen.test.server.testcase_configs import ( BaselineConfig, @@ -69,6 +70,7 @@ def test_runner_override_preserves_other_metrics(monkeypatch, runner, flux, qwen ], ) def test_runner_baseline_enforces_e2e_boundary(monkeypatch, runner): + monkeypatch.setattr(current_platform, "is_hip", lambda: False) monkeypatch.setenv("RUNNER_NAME", runner) monkeypatch.setenv("SGLANG_GEN_BASELINE", "0") monkeypatch.delenv("SGLANG_E2E_TOLERANCE", raising=False) diff --git a/python/sglang/multimodal_gen/test/unit/test_sequential_server_requests.py b/python/sglang/multimodal_gen/test/unit/test_sequential_server_requests.py index 24f54accce..3ae91c0c59 100644 --- a/python/sglang/multimodal_gen/test/unit/test_sequential_server_requests.py +++ b/python/sglang/multimodal_gen/test/unit/test_sequential_server_requests.py @@ -124,6 +124,7 @@ def harness(monkeypatch): monkeypatch.setenv("SGLANG_GEN_BASELINE", "0") monkeypatch.setenv("SGLANG_SKIP_CONSISTENCY", "0") monkeypatch.setattr(test_server_common.current_platform, "is_cuda", lambda: True) + monkeypatch.setattr(test_server_common.current_platform, "is_hip", lambda: False) scenario = ScenarioConfig( stages_ms={"DenoisingStage": 10}, denoise_step_ms={0: 5, 1: 5}, diff --git a/python/sglang/multimodal_gen/test/unit/test_suite_partitioning.py b/python/sglang/multimodal_gen/test/unit/test_suite_partitioning.py index a372d2db1e..69b9fd2bc4 100644 --- a/python/sglang/multimodal_gen/test/unit/test_suite_partitioning.py +++ b/python/sglang/multimodal_gen/test/unit/test_suite_partitioning.py @@ -81,6 +81,34 @@ def test_suite_is_fully_scheduled_for_any_shard_count(suite, total_partitions): assert sorted(scheduled_standalone_files) == sorted(expected_standalone_files) +@pytest.mark.parametrize("total_partitions", [1, 4, 8]) +def test_file_suite_runs_each_file_once(monkeypatch, tmp_path, total_partitions): + files = [f"test_{i}.py" for i in range(5)] + for filename in files: + (tmp_path / filename).touch() + monkeypatch.setitem(run_suite.FILE_SUITES, "unit", files) + executed = [] + + def run_files(assigned, filter_expr, junit_xml_path): + assert assigned + assert filter_expr == "not ltx2_vae_channels_last" + executed.extend(assigned) + return 1, [], {} + + monkeypatch.setattr(run_suite, "run_pytest", run_files) + for partition_id in range(total_partitions): + args = SimpleNamespace( + suite="unit", + partition_id=partition_id, + total_partitions=total_partitions, + filter="not ltx2_vae_channels_last", + ) + result = run_suite._run_file_suite(args, tmp_path) + assert result == (1 if partition_id < len(files) else 0) + + assert sorted(executed) == sorted(str(tmp_path / filename) for filename in files) + + def test_failing_cases_do_not_skip_the_shards_standalone_files(monkeypatch, tmp_path): """Standalone files used to own a shard, so cases could not block them.""" standalone_rel = "../single_test_file/test_disagg_server.py" diff --git a/python/sglang/multimodal_gen/test/unit/test_wan_temb_table_slices.py b/python/sglang/multimodal_gen/test/unit/test_wan_temb_table_slices.py index de54a54634..4092036f44 100644 --- a/python/sglang/multimodal_gen/test/unit/test_wan_temb_table_slices.py +++ b/python/sglang/multimodal_gen/test/unit/test_wan_temb_table_slices.py @@ -6,10 +6,14 @@ from sglang.multimodal_gen.runtime.models.dits.wanvideo import ( _eager_temb_table_slices, _wan_temb_table_slices, ) +from sglang.multimodal_gen.runtime.platforms import current_platform class TestWanTembTableSlices(unittest.TestCase): - @unittest.skipUnless(torch.cuda.is_available(), "requires CUDA") + @unittest.skipUnless( + torch.cuda.is_available() and current_platform.is_cuda(), + "requires NVIDIA CUDA fusion", + ) def test_fused_matches_eager_and_is_contiguous(self): torch.manual_seed(0) for batch, seq, hidden in [(1, 517, 3072), (2, 64, 1536)]: diff --git a/scripts/ci/npu/npu_ci_install_dependency.sh b/scripts/ci/npu/npu_ci_install_dependency.sh index c410994ac3..683d6ef6f9 100755 --- a/scripts/ci/npu/npu_ci_install_dependency.sh +++ b/scripts/ci/npu/npu_ci_install_dependency.sh @@ -79,7 +79,7 @@ ${PIP_INSTALL} torch-npu==2.10.0.post6 --extra-index-url https://ascend.devcloud case "$(arch)" in aarch64) - ${PIP_INSTALL} https://sglang-ascend.obs.cn-east-3.myhuaweicloud.com/ta/triton_ascend-3.2.2-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl + ${PIP_INSTALL} "${GITHUB_PROXY_URL}https://github.com/triton-lang/triton-ascend/releases/download/v3.2.2/triton_ascend-3.2.2-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl" ;; x86_64) ${PIP_INSTALL} "${GITHUB_PROXY_URL}https://github.com/triton-lang/triton-ascend/releases/download/v3.2.2/triton_ascend-3.2.2-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl" diff --git a/test/registered/kernels/ops/attention/test_fused_store_index_cache.py b/test/registered/kernels/ops/attention/test_fused_store_index_cache.py index a7ecb21cb9..a842a7d9b0 100644 --- a/test/registered/kernels/ops/attention/test_fused_store_index_cache.py +++ b/test/registered/kernels/ops/attention/test_fused_store_index_cache.py @@ -251,6 +251,10 @@ def test_fused_kernel_matches_own_algorithm(num_tokens: int, base_index: int): device = torch.device("cuda") key = torch.randn((num_tokens, HEAD_DIM), device=device, dtype=torch.bfloat16) + # Include a rounding tie between adjacent FP8 subnormals. + key[0].zero_() + key[0, 0] = 2.0 + key[0, 1] = 2**-15 loc = ( base_index + torch.randperm(num_tokens, device=device, dtype=torch.int64) ).contiguous() @@ -279,17 +283,14 @@ def test_fused_kernel_matches_own_algorithm(num_tokens: int, base_index: int): f"(expected < 1% from rounding tie-breaks)" ) - # 3) Where codes differ, the difference should be exactly 1 ULP. - # In FP8 E4M3: if the float-cast value is V, the adjacent value - # differs by ~V * 0.1 (relative) at most. + # 3) Compare representable steps directly: a subnormal ULP can exceed 15%. if mismatch.any(): - diff = (out_f[mismatch] - ref_f[mismatch]).abs() - rel_diff = diff / ref_f[mismatch].abs().clamp(min=1e-6) - # 1-ULP relative difference for E4M3 is at most ~12.5% (2^-3) - assert rel_diff.max().item() <= 0.15, ( - f"FP8 code difference exceeds 1-ULP: max relative diff = " - f"{rel_diff.max().item():.4f}" - ) + out_codes = out_f[mismatch].to(FP8_DTYPE).view(torch.uint8).to(torch.int16) + ref_codes = ref_f[mismatch].to(FP8_DTYPE).view(torch.uint8).to(torch.int16) + # Order the sign-magnitude codes numerically, with both zeros at 0. + out_codes = torch.where(out_codes < 128, out_codes, 128 - out_codes) + ref_codes = torch.where(ref_codes < 128, ref_codes, 128 - ref_codes) + torch.testing.assert_close(out_codes, ref_codes, rtol=0, atol=1) # 4) Dequantized values should be close. # Max error from 1-ULP: scale * fp8_ulp ≈ (abs_max/448) * 32 diff --git a/test/registered/kernels/ops/diffusion/test_activation.py b/test/registered/kernels/ops/diffusion/test_activation.py index 4a2802ad27..9ca776f2cb 100644 --- a/test/registered/kernels/ops/diffusion/test_activation.py +++ b/test/registered/kernels/ops/diffusion/test_activation.py @@ -16,9 +16,6 @@ import torch import torch.nn.functional as F from sglang.kernels.ops.diffusion import ( - can_use_fused_bias_glu, - can_use_fused_bias_silu, - can_use_fused_silu_mul, fused_bias_glu, fused_bias_silu, fused_packed_silu_mul_bitexact, @@ -40,7 +37,6 @@ def test_sana_bias_silu_is_bit_exact(channels): ).to(memory_format=torch.channels_last) bias = torch.randn(channels, device="cuda", dtype=torch.bfloat16) - assert can_use_fused_bias_silu(x, bias) actual = fused_bias_silu(x, bias) expected = F.silu(x + bias[None, :, None, None]) @@ -58,7 +54,6 @@ def test_sana_bias_glu_is_bit_exact(channels): ).to(memory_format=torch.channels_last) bias = torch.randn(2 * channels, device="cuda", dtype=torch.bfloat16) - assert can_use_fused_bias_glu(x, bias) actual = fused_bias_glu(x, bias) biased = x + bias[None, :, None, None] hidden, gate = torch.chunk(biased, 2, dim=1) @@ -82,7 +77,6 @@ def test_silu_mul_is_bit_exact(shape): a = torch.randn(shape, device="cuda", dtype=torch.bfloat16) b = torch.randn_like(a) - assert can_use_fused_silu_mul(a, b) assert torch.equal(fused_silu_mul_bitexact(a, b), F.silu(a) * b) @@ -102,11 +96,5 @@ def test_packed_silu_mul_is_bit_exact(hidden, strided): assert torch.equal(fused_packed_silu_mul_bitexact(x), expected) -def test_silu_mul_rejects_mismatched_operands(): - a = torch.randn(1, 8, 64, device="cuda", dtype=torch.bfloat16) - assert not can_use_fused_silu_mul(a, a.float()) # mixed dtypes - assert not can_use_fused_silu_mul(a, a[:, :-1]) # mismatched shapes - - if __name__ == "__main__": sys.exit(pytest.main([__file__, "-v"])) diff --git a/test/registered/kernels/ops/diffusion/test_complex_rope.py b/test/registered/kernels/ops/diffusion/test_complex_rope.py index ef6747f21f..bd480f6c3f 100644 --- a/test/registered/kernels/ops/diffusion/test_complex_rope.py +++ b/test/registered/kernels/ops/diffusion/test_complex_rope.py @@ -6,9 +6,6 @@ import torch from sglang.kernels.ops.diffusion import ( BitExactFusionGate, - can_use_fused_complex_rope, - can_use_qknorm_complex_rope_cuda, - can_use_qknorm_complex_rope_pack, fused_complex_rope, qknorm_complex_rope_cuda, qknorm_complex_rope_pack_, @@ -45,21 +42,10 @@ def inputs(shape, dtype): ) def test_complex_rope_matches_complex_multiplication(dtype, shape): x, rope = inputs(shape, dtype) - assert can_use_fused_complex_rope(x, rope) actual = fused_complex_rope(x, rope) torch.testing.assert_close(actual, reference(x, rope), atol=0, rtol=0) -def test_complex_rope_layout_guards(): - x, rope = inputs((2, 17, 3, 64), torch.bfloat16) - assert not can_use_fused_complex_rope(x.cpu(), rope.cpu()) - assert not can_use_fused_complex_rope(x.double(), rope) - assert not can_use_fused_complex_rope(x, rope.to(torch.complex128)) - assert not can_use_fused_complex_rope(x[:, ::2], rope[::2]) - assert not can_use_fused_complex_rope(x, rope[:-1]) - assert not can_use_fused_complex_rope(x[:, :0], rope[:0]) - - def test_complex_rope_compile_and_graph_replay(): x, rope = inputs((1, 257, 8, 128), torch.bfloat16) compiled = torch.compile(fused_complex_rope, fullgraph=True) @@ -136,7 +122,6 @@ def test_norm_rope_matches_eager(shape, amplitude): norm = make_norm() x = torch.randn(shape, device="cuda", dtype=torch.bfloat16) * amplitude rope = make_rope(shape[1]) - assert can_use_qknorm_complex_rope_cuda(x, norm.weight, rope) out = qknorm_complex_rope_cuda(x, norm.weight, rope, EPS) assert_bits_equal(out, eager_norm_rope(x, norm, rope)) @@ -192,9 +177,6 @@ def test_pack_matches_eager(batch, seq, prefix, heads, in_place): else: k_src, v_src = k, v q_work = q.clone() - assert can_use_qknorm_complex_rope_pack( - q_work, k_out, v_out, norm_q.weight, norm_k.weight, rope, kp, vp, k_src, v_src - ) qknorm_complex_rope_pack_( q_work, k_out, @@ -229,9 +211,6 @@ def test_pack_accepts_padded_token_stride(): q_view.copy_(q) k_out[:, prefix:].copy_(k) v_out[:, prefix:].copy_(v) - assert can_use_qknorm_complex_rope_pack( - q_view, k_out, v_out, norm_q.weight, norm_k.weight, rope, kp, vp, None, None - ) qknorm_complex_rope_pack_( q_view, k_out, @@ -250,70 +229,6 @@ def test_pack_accepts_padded_token_stride(): assert_bits_equal(v_out, torch.cat([vp, v], dim=1)) -@torch.no_grad() -def test_predicates_reject_unsupported_layouts(): - norm = make_norm() - x = torch.randn(1, 17, 4, HEAD_DIM, device="cuda", dtype=torch.bfloat16) - rope = make_rope(17) - assert can_use_qknorm_complex_rope_cuda(x, norm.weight, rope) - assert not can_use_qknorm_complex_rope_cuda(x.cpu(), norm.weight.cpu(), rope.cpu()) - assert not can_use_qknorm_complex_rope_cuda(x.half(), norm.weight.half(), rope) - assert not can_use_qknorm_complex_rope_cuda( - x[..., :64], norm.weight[:64], rope[:, :32] - ) - assert not can_use_qknorm_complex_rope_cuda(x.transpose(1, 2), norm.weight, rope) - assert not can_use_qknorm_complex_rope_cuda(x, norm.weight, rope[:-1]) - assert not can_use_qknorm_complex_rope_cuda( - x, norm.weight, rope.to(torch.complex128) - ) - assert not can_use_qknorm_complex_rope_cuda(x, norm.weight.float(), rope) - k_out = torch.empty(1, 17 + 3, 4, HEAD_DIM, device="cuda", dtype=torch.bfloat16) - kp = torch.randn(1, 3, 4, HEAD_DIM, device="cuda", dtype=torch.bfloat16) - assert can_use_qknorm_complex_rope_pack( - x, k_out, k_out.clone(), norm.weight, norm.weight, rope, kp, kp, None, None - ) - # wrong prefix + seq length, batch mismatch, non-contiguous prefix, short K source - assert not can_use_qknorm_complex_rope_pack( - x, - k_out[:, :-1], - k_out.clone(), - norm.weight, - norm.weight, - rope, - kp, - kp, - None, - None, - ) - assert not can_use_qknorm_complex_rope_pack( - x, - k_out, - k_out.clone(), - norm.weight, - norm.weight, - rope, - kp.expand(2, -1, -1, -1), - kp, - None, - None, - ) - assert not can_use_qknorm_complex_rope_pack( - x, - k_out, - k_out.clone(), - norm.weight, - norm.weight, - rope, - kp.transpose(1, 2), - kp, - None, - None, - ) - assert not can_use_qknorm_complex_rope_pack( - x, k_out, k_out.clone(), norm.weight, norm.weight, rope, kp, kp, x[:, :-1], None - ) - - @torch.no_grad() def test_pack_graph_capture_and_replay(): batch, seq, prefix, heads = 1, 65, 3, 8 @@ -369,5 +284,73 @@ def test_pack_graph_capture_and_replay(): assert_bits_equal(v_out, torch.cat([vp2, v2], dim=1)) +@torch.no_grad() +@pytest.mark.parametrize( + "cuda_disabled,triton_disabled", [(False, False), (True, False), (True, True)] +) +def test_model_qknorm_rope_accepts_packed_projection_views( + monkeypatch, cuda_disabled, triton_disabled +): + packed = torch.randn(2, 17, 3, 4, HEAD_DIM, device="cuda", dtype=torch.bfloat16) + x = packed.unbind(2)[0] + assert not x.is_contiguous() + norm, rope = make_norm(), make_rope(17) + for name, disabled in ( + ("_QK_ROPE_CUDA_FUSION", cuda_disabled), + ("_QK_ROPE_FUSION", triton_disabled), + ("_ROPE_FUSION", False), + ): + gate = BitExactFusionGate(name) + gate.disabled = disabled + monkeypatch.setattr(qwen_image21, name, gate) + expected = reference(norm(x), rope) + actual = qwen_image21.apply_qk_norm_rope(x, norm, rope) + assert torch.equal(actual, expected) + assert actual.is_contiguous() + + +@torch.no_grad() +def test_model_pack_verifies_strided_projection_views(monkeypatch): + from types import SimpleNamespace + + batch, seq, prefix, heads = 1, 17, 3, 4 + packed = torch.randn( + batch, prefix + seq, 3, heads, HEAD_DIM, device="cuda", dtype=torch.bfloat16 + ) + q = packed[:, prefix:, 0] + k_out, v_out = packed[:, :, 1], packed[:, :, 2] + k, v = k_out[:, prefix:], v_out[:, prefix:] + kp, vp = torch.randn_like(q[:, :prefix]), torch.randn_like(q[:, :prefix]) + norm_q, norm_k, rope = make_norm(), make_norm(), make_rope(seq) + attention = SimpleNamespace(head_dim=HEAD_DIM, norm_q=norm_q, norm_k=norm_k) + expected = ( + reference(norm_q(q), rope), + torch.cat((kp, reference(norm_k(k), rope)), 1), + torch.cat((vp, v), 1), + ) + gate = BitExactFusionGate("test packed projection verification") + monkeypatch.setattr(qwen_image21, "_KV_PACK_CUDA_FUSION", gate) + actual = qwen_image21.QwenImage21Attention._pack_kv_cuda( + attention, q, k, v, rope, kp, vp, k_out, v_out + ) + assert gate.verified + for got, want in zip(actual, expected, strict=True): + assert torch.equal(got, want) + + +@torch.no_grad() +def test_compiled_native_norm_rope_preserves_output_layout(): + # Dense transposes differ from packed column views: empty_like would retain + # their strides, but the out-of-place kernel writes contiguous output. + x = torch.randn(17, 2, 4, HEAD_DIM, device="cuda", dtype=torch.bfloat16).transpose( + 0, 1 + ) + norm, rope = make_norm(), make_rope(17) + compiled = torch.compile(qknorm_complex_rope_cuda, fullgraph=True) + actual = compiled(x, norm.weight, rope, EPS) + assert torch.equal(actual, reference(norm(x), rope)) + assert actual.is_contiguous() + + if __name__ == "__main__": sys.exit(pytest.main([__file__, "-v"])) diff --git a/test/registered/kernels/ops/diffusion/test_helios_qk_rope.py b/test/registered/kernels/ops/diffusion/test_helios_qk_rope.py index 271719d33a..17b304801c 100644 --- a/test/registered/kernels/ops/diffusion/test_helios_qk_rope.py +++ b/test/registered/kernels/ops/diffusion/test_helios_qk_rope.py @@ -6,7 +6,6 @@ import torch import torch.nn as nn from sglang.kernels.ops.diffusion import ( - can_use_helios_qk_rope, fused_inplace_helios_qk_rope, ) from sglang.test.ci.ci_register import register_cuda_ci @@ -25,7 +24,7 @@ def _reference(value: torch.Tensor, freqs: torch.Tensor) -> torch.Tensor: return out.type_as(value) -@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) +@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32]) @pytest.mark.parametrize( "tokens,heads,head_dim", [ @@ -63,31 +62,6 @@ def test_helios_qk_rope_matches_eager_transposed_path( assert torch.equal(k_out, k_ref) -def test_helios_qk_rope_runtime_guards() -> None: - q = torch.randn(1, 17, 8, 128, device="cuda", dtype=torch.bfloat16) - k = torch.randn_like(q) - freqs = torch.randn(1, 17, 256, device="cuda", dtype=torch.float32) - assert can_use_helios_qk_rope(q, k, freqs) - assert not can_use_helios_qk_rope(q.float(), k, freqs) - assert not can_use_helios_qk_rope(q, k.float(), freqs) - assert not can_use_helios_qk_rope(q, k, freqs.bfloat16()) - assert not can_use_helios_qk_rope(q.cpu(), k.cpu(), freqs.cpu()) - assert not can_use_helios_qk_rope(q, k, freqs[..., :-2]) - assert not can_use_helios_qk_rope(q[:, :, :, ::2], k, freqs) - assert not can_use_helios_qk_rope(q[:, :0], k[:, :0], freqs[:, :0]) - - q_unaligned = torch.empty(q.numel() + 1, device=q.device, dtype=q.dtype)[ - 1: - ].view_as(q) - k_unaligned = torch.empty(k.numel() + 1, device=k.device, dtype=k.dtype)[ - 1: - ].view_as(k) - assert q_unaligned.is_contiguous() and q_unaligned.storage_offset() == 1 - assert k_unaligned.is_contiguous() and k_unaligned.storage_offset() == 1 - assert not can_use_helios_qk_rope(q_unaligned, k, freqs) - assert not can_use_helios_qk_rope(q, k_unaligned, freqs) - - def test_helios_attention_dispatch_and_tp_fallback() -> None: import sglang.multimodal_gen.runtime.models.dits.helios as helios @@ -99,16 +73,15 @@ def test_helios_attention_dispatch_and_tp_fallback() -> None: freqs = torch.randn(1, 17, 256, device="cuda", dtype=torch.float32) with ( - patch.object(helios, "can_use_helios_qk_rope", return_value=True), patch.object(helios, "fused_inplace_helios_qk_rope") as fused, ): q_out, k_out = attention._apply_rotary_qk(q, k, freqs) assert q_out is q assert k_out is k fused.assert_called_once() - assert fused.call_args.args[0].shape == (17, 8, 128) - assert fused.call_args.args[1].shape == (17, 8, 128) - assert fused.call_args.args[2].shape == (17, 256) + assert fused.call_args.args[0] is q + assert fused.call_args.args[1] is k + assert fused.call_args.args[2] is freqs attention.tp_rmsnorm = True with patch.object(helios, "fused_inplace_helios_qk_rope") as fused: @@ -158,6 +131,17 @@ def test_helios_attention_fullgraph_dispatch() -> None: assert torch.equal(k_out, k_ref) +@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32]) +def test_helios_qk_rope_batched(dtype): + q = torch.randn(2, 17, 8, 128, device="cuda", dtype=dtype) + k = torch.randn_like(q) + freqs = torch.randn(2, 17, 256, device="cuda", dtype=torch.float32) + q_ref, k_ref = _reference(q, freqs), _reference(k, freqs) + fused_inplace_helios_qk_rope(q, k, freqs) + assert torch.equal(q, q_ref) + assert torch.equal(k, k_ref) + + def test_helios_qk_rope_rejects_bad_frequency_shape() -> None: q = torch.randn(3, 2, 128, device="cuda", dtype=torch.bfloat16) k = torch.randn_like(q) diff --git a/test/registered/kernels/ops/diffusion/test_interleaved_rope_fp64.py b/test/registered/kernels/ops/diffusion/test_interleaved_rope_fp64.py index 5c77c34851..89620f7de2 100644 --- a/test/registered/kernels/ops/diffusion/test_interleaved_rope_fp64.py +++ b/test/registered/kernels/ops/diffusion/test_interleaved_rope_fp64.py @@ -5,7 +5,6 @@ import torch from sglang.kernels.jit.utils import get_ci_test_range from sglang.kernels.ops.diffusion import ( - can_use_interleaved_rope_fp64, fused_interleaved_rope_fp64, ) from sglang.test.ci.ci_register import register_cuda_ci @@ -75,24 +74,5 @@ def test_interleaved_rope_fp64_is_bit_exact( assert k_out.data_ptr() not in (q.data_ptr(), k.data_ptr()) -def test_interleaved_rope_fp64_predicate_rejects_unsupported_inputs() -> None: - q = torch.empty(1, 17, 3, 12, dtype=torch.bfloat16, device="cuda") - k = torch.empty_like(q) - cos = torch.empty(1, 17, 1, 12, dtype=torch.float64, device="cuda") - sin = torch.empty_like(cos) - - assert can_use_interleaved_rope_fp64(q, k, cos, sin) - assert not can_use_interleaved_rope_fp64(q.flatten(), k, cos, sin) - assert not can_use_interleaved_rope_fp64(q.float(), k, cos, sin) - assert not can_use_interleaved_rope_fp64(q, k[..., ::2], cos, sin) - assert not can_use_interleaved_rope_fp64(q, k, cos.float(), sin) - assert not can_use_interleaved_rope_fp64(q, k, cos[..., :-2], sin[..., :-2]) - unaligned = torch.empty(q.numel() + 1, dtype=torch.bfloat16, device="cuda")[ - 1: - ].view_as(q) - assert unaligned.is_contiguous() - assert not can_use_interleaved_rope_fp64(unaligned, k, cos, sin) - - if __name__ == "__main__": sys.exit(pytest.main([__file__, "-v", "-s"])) diff --git a/test/registered/kernels/ops/diffusion/test_joint_qkv_cat.py b/test/registered/kernels/ops/diffusion/test_joint_qkv_cat.py index ff254d0cc7..5c72366a89 100644 --- a/test/registered/kernels/ops/diffusion/test_joint_qkv_cat.py +++ b/test/registered/kernels/ops/diffusion/test_joint_qkv_cat.py @@ -3,7 +3,7 @@ from unittest.mock import patch import torch -from sglang.kernels.ops.diffusion import can_use_joint_qkv_cat, joint_qkv_cat +from sglang.kernels.ops.diffusion import joint_qkv_cat from sglang.kernels.ops.diffusion.sites.bitexact_gate import BitExactFusionGate from sglang.multimodal_gen.runtime.models.dits import joy_image from sglang.test.ci.ci_register import register_cuda_ci @@ -46,7 +46,6 @@ class TestJointQKVCat(CustomTestCase): for value in inputs: value.view(torch.int16)[0, 0, 0, :8].copy_(bits) before = tuple(x.clone() for x in inputs) - self.assertTrue(can_use_joint_qkv_cat(*inputs)) out = joint_qkv_cat(*inputs) self.assert_bits_equal(out, reference(inputs)) self.assert_bits_equal( diff --git a/test/registered/kernels/ops/diffusion/test_layernorm_modulate.py b/test/registered/kernels/ops/diffusion/test_layernorm_modulate.py index 03c614a4d7..f8de645747 100644 --- a/test/registered/kernels/ops/diffusion/test_layernorm_modulate.py +++ b/test/registered/kernels/ops/diffusion/test_layernorm_modulate.py @@ -6,7 +6,6 @@ import torch from torch.nn import functional as F from sglang.kernels.ops.diffusion import ( - can_use_fused_layernorm_modulate, fused_layernorm_modulate, ) from sglang.test.ci.ci_register import register_cuda_ci @@ -35,7 +34,6 @@ def test_modulation_preserves_bits(shape, amplitude, eps, has_shift): scale[:, :3] = torch.tensor([-1, 0, 1], device=x.device, dtype=x.dtype) if not has_shift: shift = None - assert can_use_fused_layernorm_modulate(x, scale, shift) actual = fused_layernorm_modulate(x, scale, shift, eps) expected = reference(x, scale, shift, eps) assert torch.equal(actual.view(torch.int16), expected.view(torch.int16)) @@ -50,21 +48,6 @@ def test_scale_only_preserves_signed_zero(): assert torch.equal(actual.view(torch.int16), expected.view(torch.int16)) -def test_scale_only_layout_guards(): - x = torch.randn(2, 17, 128, device="cuda", dtype=torch.bfloat16) - scale = torch.randn(2, 128, device=x.device, dtype=x.dtype) - assert can_use_fused_layernorm_modulate(x, scale, None) - assert not can_use_fused_layernorm_modulate(x.cpu(), scale.cpu(), None) - assert not can_use_fused_layernorm_modulate(x.float(), scale.float(), None) - assert not can_use_fused_layernorm_modulate(x[:, ::2], scale, None) - assert not can_use_fused_layernorm_modulate(x, scale.float(), None) - assert not can_use_fused_layernorm_modulate(x, scale[:, :-1], None) - assert not can_use_fused_layernorm_modulate(x[:, :0], scale, None) - assert not can_use_fused_layernorm_modulate(x, scale, scale.float()) - strided = torch.empty(2, 256, device=x.device, dtype=x.dtype)[:, :128] - assert not can_use_fused_layernorm_modulate(x, scale, strided) - - @pytest.mark.parametrize("has_shift", [False, True]) def test_compile_and_graph_replay(has_shift): x = torch.randn(2, 17, 128, device="cuda", dtype=torch.bfloat16) diff --git a/test/registered/kernels/ops/diffusion/test_layout.py b/test/registered/kernels/ops/diffusion/test_layout.py index ef11183ffa..a12c64f58c 100644 --- a/test/registered/kernels/ops/diffusion/test_layout.py +++ b/test/registered/kernels/ops/diffusion/test_layout.py @@ -104,15 +104,13 @@ def test_usp_merge_heads_bitwise(dtype, world, seq, batch, h_local, head_dim): def test_usp_merge_heads_unsupported_inputs_use_exact_fallback(): # The wrapper degrades to the aten permute for anything the fast path - # rejects -- a wrong rank, a transposed view, an empty leading dim, or a - # ROCm build -- so callers never need their own guard. + # rejects: a transposed view, an empty leading dim, or a ROCm build. x = torch.randn(2, 4, 1, 4, 64, dtype=torch.bfloat16, device=DEVICE) - for value in (x.transpose(0, 1), x[:0], x[0]): + for value in (x.transpose(0, 1), x[:0]): assert not can_use_usp_merge_heads(value) - if value.dim() == 5: - assert torch.equal( - usp_merge_heads(value), value.permute(2, 1, 0, 3, 4).contiguous() - ) + assert torch.equal( + usp_merge_heads(value), value.permute(2, 1, 0, 3, 4).contiguous() + ) with patch.object(torch.version, "hip", "6.3"): assert not can_use_usp_merge_heads(x) diff --git a/test/registered/kernels/ops/diffusion/test_ltx25_decoder_rope.py b/test/registered/kernels/ops/diffusion/test_ltx25_decoder_rope.py index f4ba23a52a..284eccde66 100644 --- a/test/registered/kernels/ops/diffusion/test_ltx25_decoder_rope.py +++ b/test/registered/kernels/ops/diffusion/test_ltx25_decoder_rope.py @@ -5,7 +5,6 @@ import torch from sglang.kernels.jit.utils import get_ci_test_range from sglang.kernels.ops.diffusion import ( - can_use_ltx25_decoder_rope, fused_ltx25_decoder_rope, ) from sglang.test.ci.ci_register import register_cuda_ci @@ -91,24 +90,5 @@ def test_ltx25_decoder_rope_is_bit_exact( assert k_out.data_ptr() not in (q.data_ptr(), k.data_ptr()) -def test_ltx25_decoder_rope_predicate_rejects_unsupported_inputs() -> None: - q = torch.empty(1, 3, 7, 7, 1, 64, dtype=torch.bfloat16, device="cuda") - k = torch.empty_like(q) - tables = make_tables(3, 7, 7) - - assert can_use_ltx25_decoder_rope(q, k, tables, DIM_SPLIT) - assert not can_use_ltx25_decoder_rope(q.flatten(), k, tables, DIM_SPLIT) - assert not can_use_ltx25_decoder_rope(q.float(), k, tables, DIM_SPLIT) - assert not can_use_ltx25_decoder_rope(q, k[..., ::2], tables, DIM_SPLIT) - assert not can_use_ltx25_decoder_rope(q, k, tables, (16, 16, 16)) - bad_tables = (tables[0], tables[1], (tables[2][0].double(), tables[2][1])) - assert not can_use_ltx25_decoder_rope(q, k, bad_tables, DIM_SPLIT) - unaligned = torch.empty(q.numel() + 1, dtype=torch.bfloat16, device="cuda")[ - 1: - ].view_as(q) - assert unaligned.is_contiguous() - assert not can_use_ltx25_decoder_rope(unaligned, k, tables, DIM_SPLIT) - - if __name__ == "__main__": sys.exit(pytest.main([__file__, "-v", "-s"])) diff --git a/test/registered/kernels/ops/diffusion/test_model_fast_paths.py b/test/registered/kernels/ops/diffusion/test_model_fast_paths.py index b2934ee3bb..128b3e1144 100644 --- a/test/registered/kernels/ops/diffusion/test_model_fast_paths.py +++ b/test/registered/kernels/ops/diffusion/test_model_fast_paths.py @@ -41,9 +41,6 @@ import sglang.multimodal_gen.runtime.models.dits.qwen_image21 as qwen_image21 import sglang.multimodal_gen.runtime.models.dits.sana as sana from sglang.kernels.ops.diffusion import ( BitExactFusionGate, - can_use_fused_layernorm_modulate, - can_use_fused_qk_head_layernorm, - can_use_fused_rmsnorm_scale_shift, fused_ltx2_rms_norm_modulate, hunyuan_qkv_rope_pack, mark_fused_ln_modulate_site, @@ -246,21 +243,6 @@ def test_packed_sdpa_preserves_training_dropout_and_missing_api_fallbacks(): assert forward.call_count == 2 -def test_bitexact_norm_guards_follow_platform(): - # Runs on both lanes, with shapes inside every guard's contract so only the - # platform decides: engaged on CUDA, rejected on ROCm. A fatal LLVM error - # there kills the process, so the sites' own try/except cannot be what - # catches it -- the guards have to. - x = torch.randn(1, 256, 4096, device="cuda", dtype=torch.bfloat16) - row = torch.randn(1, 4096, device="cuda", dtype=torch.bfloat16) - vec = torch.randn(1, 1, 4096, device="cuda", dtype=torch.bfloat16) - weight = torch.randn(4096, device="cuda", dtype=torch.bfloat16) - q = torch.randn(1, 256, 32, 128, device="cuda", dtype=torch.bfloat16) - assert can_use_fused_layernorm_modulate(x, row, row) is is_cuda() - assert can_use_fused_qk_head_layernorm(q, q) is is_cuda() - assert can_use_fused_rmsnorm_scale_shift(x, weight, vec, vec) is is_cuda() - - # ------------------------------------------------------------------------- # Qwen-Image -- final LayerNorm + adaLN scale/shift # ------------------------------------------------------------------------- diff --git a/test/registered/kernels/ops/diffusion/test_rmsnorm_preserve_reduction.py b/test/registered/kernels/ops/diffusion/test_rmsnorm_preserve_reduction.py index aac5659801..1e31be8d5e 100644 --- a/test/registered/kernels/ops/diffusion/test_rmsnorm_preserve_reduction.py +++ b/test/registered/kernels/ops/diffusion/test_rmsnorm_preserve_reduction.py @@ -5,7 +5,6 @@ import pytest import torch from sglang.kernels.ops.diffusion import ( - can_use_rmsnorm_preserve_reduction, rmsnorm_preserve_reduction, ) from sglang.test.ci.ci_register import register_cuda_ci @@ -31,27 +30,26 @@ def test_preserves_native_reduction_and_rounding(dtype, shape, scale, eps): torch.manual_seed(42) x = (torch.randn(shape, device="cuda") * scale).to(dtype) weight = torch.randn(shape[-1], device="cuda", dtype=dtype) - assert can_use_rmsnorm_preserve_reduction(x, weight) + actual = rmsnorm_preserve_reduction(x, weight, eps) torch.testing.assert_close(actual, reference(x, weight, eps), atol=0, rtol=0) -def test_layout_guards_and_offset(): - x = torch.randn(259, 128, device="cuda", dtype=torch.bfloat16)[2:] - weight = torch.randn(128, device="cuda", dtype=x.dtype) - assert can_use_rmsnorm_preserve_reduction(x, weight) - torch.testing.assert_close( - rmsnorm_preserve_reduction(x, weight, 1e-6), - reference(x, weight, 1e-6), - atol=0, - rtol=0, - ) - assert not can_use_rmsnorm_preserve_reduction(x.cpu(), weight.cpu()) - assert not can_use_rmsnorm_preserve_reduction(x.float(), weight.float()) - assert not can_use_rmsnorm_preserve_reduction(x[:, ::2], weight[::2]) - assert not can_use_rmsnorm_preserve_reduction(x, weight.float()) - assert not can_use_rmsnorm_preserve_reduction(x, weight[:-1]) - assert not can_use_rmsnorm_preserve_reduction(x[:0], weight) +def test_packed_projection_view(): + packed = torch.randn(2, 17, 3, 4, 128, device="cuda", dtype=torch.bfloat16) + x = packed.unbind(2)[0] + weight = torch.randn(128, device=x.device, dtype=x.dtype) + actual = rmsnorm_preserve_reduction(x, weight, 1e-6) + assert torch.equal(actual, reference(x, weight, 1e-6)) + + +def test_compiled_transposed_input_has_contiguous_output(): + x = torch.randn(17, 2, 128, device="cuda", dtype=torch.bfloat16).transpose(0, 1) + weight = torch.randn(128, device=x.device, dtype=x.dtype) + compiled = torch.compile(rmsnorm_preserve_reduction, fullgraph=True) + actual = compiled(x, weight, 1e-6) + assert torch.equal(actual, reference(x, weight, 1e-6)) + assert actual.is_contiguous() def test_compile_and_graph_replay(): diff --git a/test/registered/kernels/ops/diffusion/test_sana_wm_conv_post.py b/test/registered/kernels/ops/diffusion/test_sana_wm_conv_post.py index 1f33b86258..0e5fe675b0 100644 --- a/test/registered/kernels/ops/diffusion/test_sana_wm_conv_post.py +++ b/test/registered/kernels/ops/diffusion/test_sana_wm_conv_post.py @@ -9,8 +9,6 @@ import torch.nn.functional as F import sglang.multimodal_gen.runtime.models.dits.sana_wm_components as wm from sglang.kernels.ops.diffusion import BitExactFusionGate from sglang.kernels.ops.diffusion.activation.sana_conv_post_triton import ( - can_use_fused_bias_glu, - can_use_fused_bias_silu, fused_bias_glu, fused_bias_silu, ) @@ -45,11 +43,6 @@ class TestSanaWMConvPost(CustomTestCase): actual = fused_bias_glu(x, b) self.assertTrue(torch.equal(actual, a * F.silu(g))) self.assertTrue(actual.is_contiguous(memory_format=layout)) - # Use a real spatial slice; width-one tensors remain contiguous. - sliced = torch.empty(2, 34, 7, 10, device="cuda", dtype=x.dtype)[:, :, :, ::2] - self.assertFalse(can_use_fused_bias_silu(sliced, bias)) - self.assertFalse(can_use_fused_bias_glu(sliced, None)) - self.assertFalse(can_use_fused_bias_glu(x.float(), None)) @staticmethod def reference(module, x): diff --git a/test/registered/kernels/ops/diffusion/test_vdn_delta_factors.py b/test/registered/kernels/ops/diffusion/test_vdn_delta_factors.py index f5451c274b..33647d17bb 100644 --- a/test/registered/kernels/ops/diffusion/test_vdn_delta_factors.py +++ b/test/registered/kernels/ops/diffusion/test_vdn_delta_factors.py @@ -6,7 +6,7 @@ import sys import pytest import torch -from sglang.kernels.ops.diffusion import can_use_vdn_delta_factors, vdn_delta_factors +from sglang.kernels.ops.diffusion import vdn_delta_factors from sglang.multimodal_gen.runtime.models.dits import minimax_h3_vdn as vdn from sglang.test.ci.ci_register import register_cuda_ci @@ -55,7 +55,6 @@ def _fp64(A, B, alpha): @pytest.mark.parametrize("beta_scale", [1.0, 50.0]) def test_matches_eager_and_fp64(frames, heads, tokens, beta_scale): A, B, alpha = _inputs(frames, heads, tokens, beta_scale) - assert can_use_vdn_delta_factors(A, B, alpha) t_ref, j_ref = _fp64(A, B, alpha) t_eager, j_eager = vdn.delta_factor_apply( "vdn_solve", alpha, A, B, tokens_per_frame=tokens @@ -110,25 +109,10 @@ def test_storage_offset_input_matches_aligned(which): ref = vdn_delta_factors(A, B, alpha) inputs = {"A": A, "B": B, "alpha": alpha} inputs[which] = _storage_offset_copy(inputs[which]) - assert can_use_vdn_delta_factors(inputs["A"], inputs["B"], inputs["alpha"]) out = vdn_delta_factors(inputs["A"], inputs["B"], inputs["alpha"]) for got, want in zip(out, ref): assert torch.equal(got, want) -def test_can_use_rejects_unsupported(): - A, B, alpha = _inputs(2, 2, 32) - assert can_use_vdn_delta_factors(A, B, alpha) - assert not can_use_vdn_delta_factors( - A[..., :64, :64].contiguous(), - B[..., :64, :64].contiguous(), - alpha[..., :64].contiguous(), - ) - assert not can_use_vdn_delta_factors(A.bfloat16(), B, alpha) - assert not can_use_vdn_delta_factors(A.transpose(-1, -2), B, alpha) - assert not can_use_vdn_delta_factors(A, B, alpha[..., :1].expand_as(alpha)) - assert not can_use_vdn_delta_factors(A.cpu(), B.cpu(), alpha.cpu()) - - if __name__ == "__main__": sys.exit(pytest.main([__file__, "-v", "-s"])) diff --git a/test/registered/unit/disaggregation/test_register_to_bootstrap.py b/test/registered/unit/disaggregation/test_register_to_bootstrap.py index 7a39fefbbe..322bca9e66 100644 --- a/test/registered/unit/disaggregation/test_register_to_bootstrap.py +++ b/test/registered/unit/disaggregation/test_register_to_bootstrap.py @@ -33,7 +33,11 @@ class TestRegisterToBootstrap(CustomTestCase): for force_query in (False, True): for dp_rank in (0, 1): with self.subTest(force_query=force_query, dp_rank=dp_rank): - mgr = MagicMock(is_dummy_cp_rank=False, attn_dp_rank=dp_rank) + mgr = MagicMock( + is_dummy_cp_rank=False, + attn_dp_rank=dp_rank, + deferred_bootstrap=None, + ) sender = MagicMock(spec=CommonKVSender) sender._register_prefill_dp_rank = ( CommonKVSender._register_prefill_dp_rank.__get__(sender) @@ -55,8 +59,6 @@ class TestRegisterToBootstrap(CustomTestCase): mgr=mgr, bootstrap_addr="127.0.0.1:8765", bootstrap_room=4, - dest_tp_ranks=[0], - pp_rank=0, ) self.assertEqual(