diff --git a/python/sglang/kernels/jit/csrc/elementwise/qknorm.cuh b/python/sglang/kernels/jit/csrc/elementwise/qknorm.cuh index b2d862d561..dbd2ac67d2 100644 --- a/python/sglang/kernels/jit/csrc/elementwise/qknorm.cuh +++ b/python/sglang/kernels/jit/csrc/elementwise/qknorm.cuh @@ -67,6 +67,69 @@ __global__ void fused_qknorm_warp(const QKNormParams __grid_constant__ params) { PDLTriggerSecondary(); // launch secondary kernel } +// Head_dim-128 variant of fused_qknorm_warp: two rows per warp, 16 lanes x 16 B per row. +// Each lane replays the warp kernel's butterfly on two partial sums, so the output is bit-identical. +template +__global__ void fused_qknorm_half_warp(const QKNormParams __grid_constant__ params) { + using namespace device; + + constexpr int64_t kHeadDim = 128; + constexpr uint32_t kRowLanes = 16; + using Packed = packed_t; + constexpr uint32_t kPacks = kHeadDim / (2 * kRowLanes); + using Storage = AlignedVector; + + static_assert(sizeof(Float) == 2, "Only support FP16/BF16"); + const auto& [q, k, q_stride, k_stride, num_qo_heads, num_kv_heads, eps, q_weight, k_weight, num_tokens] = params; + + const auto lane_id = threadIdx.x % kWarpThreads; + const auto row_lane = lane_id % kRowLanes; + const auto num_workers = gridDim.x * kWarpsPerBlock; + const auto num_q_and_k_heads = num_qo_heads + num_kv_heads; + const auto num_rows = num_q_and_k_heads * num_tokens; + const auto num_pairs = (num_rows + 1) / 2; + const auto start_worker_id = blockIdx.x * kWarpsPerBlock + threadIdx.x / kWarpThreads; + + PDLWaitPrimary(); // wait for primary kernel + + for (auto pair = start_worker_id; pair < num_pairs; pair += num_workers) { + // a missing second row recomputes the last one and skips the store + const auto wanted_row = 2 * pair + lane_id / kRowLanes; + const auto row = wanted_row < num_rows ? wanted_row : num_rows - 1; + const int64_t token_id = row / num_q_and_k_heads; + const int64_t head_id = row % num_q_and_k_heads; + const auto load_q = head_id < num_qo_heads; + const auto input = load_q ? pointer::offset(q, 2 * (token_id * q_stride + head_id * kHeadDim)) + : pointer::offset(k, 2 * (token_id * k_stride + head_id * kHeadDim)); + const auto weight = load_q ? q_weight : k_weight; + auto vec = load_as(input, row_lane); + const auto weight_vec = load_as(weight, row_lane); + + // part[h] is the sum of warp-kernel lane 2 * row_lane + h, and reduce_sum below + // is that kernel's 16..2 xor steps; its final xor-1 step adds the pair. + float part[2] = {0.0f, 0.0f}; +#pragma unroll + for (auto i = 0u; i < kPacks; ++i) { + const auto x = cast(vec[i]); + part[i / 2] += x.x * x.x; + part[i / 2] += x.y * x.y; + } + part[0] = warp::reduce_sum(part[0]); + part[1] = warp::reduce_sum(part[1]); + const float norm_factor = math::rsqrt((part[0] + part[1]) / kHeadDim + eps); + +#pragma unroll + for (auto i = 0u; i < kPacks; ++i) { + const auto x = cast(vec[i]); + const auto w = cast(weight_vec[i]); + vec[i] = cast({x.x * norm_factor * w.x, x.y * norm_factor * w.y}); + } + if (wanted_row < num_rows) store_as(input, vec, row_lane); + } + + PDLTriggerSecondary(); // launch secondary kernel +} + // For CTA level, used for head_dim > 256 (512,1024) template __global__ void fused_qknorm_cta(const QKNormParams __grid_constant__ params) { @@ -102,12 +165,23 @@ __global__ void fused_qknorm_cta(const QKNormParams __grid_constant__ params) { PDLTriggerSecondary(); // launch secondary kernel } +// Head_dims that fused_qknorm_half_warp covers +template +inline constexpr bool kUseHalfWarp = +#ifdef USE_ROCM + false; +#else + kHeadDim == 128; +#endif + // Warp-level kernel struct for head_dim <= 256 -template +template struct QKNormKernelWarp { static_assert(std::is_same_v || std::is_same_v); static_assert(!host::norm::should_use_cta(), "Use QKNormKernelCTA for head_dim > 256"); static constexpr auto kernel = fused_qknorm_warp; + // kAllowHalfWarp = false keeps the one-row-per-warp kernel, which tests compare against + static constexpr bool kHalfWarp = kAllowHalfWarp && kUseHalfWarp; static void run(const tvm::ffi::TensorView q, @@ -161,17 +235,33 @@ struct QKNormKernelWarp { .num_tokens = num_tokens, }; - static const uint32_t max_occupancy = runtime::get_blocks_per_sm(kernel, kThreadsPerBlock); - static const uint32_t kNumSM = runtime::get_sm_count(device.unwrap().device_id); + const auto num_rows = (num_qo_heads + num_kv_heads) * num_tokens; + if constexpr (kHalfWarp) { + // the half-warp kernel moves 16 B per lane, the warp kernel only 8 B + const auto address_bits = reinterpret_cast(q.data_ptr()) | reinterpret_cast(k.data_ptr()) | + reinterpret_cast(q_weight.data_ptr()) | + reinterpret_cast(k_weight.data_ptr()) | + static_cast(2 * (Sq.unwrap() | Sk.unwrap())); + if (address_bits % 16 == 0) { + return launch>(params, div_ceil(num_rows, 2u), device.unwrap()); + } + } + launch(params, num_rows, device.unwrap()); + } + + template + static void launch(const QKNormParams& params, uint32_t num_works, DLDevice device) { + using namespace host; + + static const uint32_t max_occupancy = runtime::get_blocks_per_sm(kKernel, kThreadsPerBlock); + static const uint32_t kNumSM = runtime::get_sm_count(device.device_id); - // choose kernel based on dtype - const auto num_works = (num_qo_heads + num_kv_heads) * num_tokens; const auto needed_blocks = div_ceil(num_works, kWarpsPerBlock); // we use persistent kernel, which limit the number of blocks to reduce overhead const auto num_blocks = std::min(kNumSM * max_occupancy, needed_blocks); - LaunchKernel(num_blocks, kThreadsPerBlock, device.unwrap()) // - .enable_pdl(kUsePDL)(kernel, params); + LaunchKernel(num_blocks, kThreadsPerBlock, device) // + .enable_pdl(kUsePDL)(kKernel, params); } }; @@ -248,10 +338,10 @@ struct QKNormKernelCTA { }; // Unified dispatch: select warp or CTA kernel based on head_dim -template +template using QKNormKernel = std::conditional_t< host::norm::should_use_cta(), QKNormKernelCTA, - QKNormKernelWarp>; + QKNormKernelWarp>; } // namespace sglang diff --git a/python/sglang/kernels/ops/layernorm/norm.py b/python/sglang/kernels/ops/layernorm/norm.py index 6b961cb8eb..9a965dd1ec 100644 --- a/python/sglang/kernels/ops/layernorm/norm.py +++ b/python/sglang/kernels/ops/layernorm/norm.py @@ -21,8 +21,12 @@ logger = logging.getLogger(__name__) @cache_once -def _jit_qknorm_module(head_dim: int, dtype: torch.dtype) -> Module: - args = make_cpp_args(head_dim, is_arch_support_pdl(), dtype) +def _jit_qknorm_module( + head_dim: int, dtype: torch.dtype, half_warp: bool = True +) -> Module: + """``half_warp=False`` keeps the one-row-per-warp kernel where the two-rows + variant would apply; tests use it as the bit-exactness oracle.""" + args = make_cpp_args(head_dim, is_arch_support_pdl(), dtype, half_warp) return load_jit( "qknorm", *args, diff --git a/test/registered/kernels/ops/layernorm/test_qknorm.py b/test/registered/kernels/ops/layernorm/test_qknorm.py index 093172a998..cb0f68fee5 100644 --- a/test/registered/kernels/ops/layernorm/test_qknorm.py +++ b/test/registered/kernels/ops/layernorm/test_qknorm.py @@ -82,5 +82,31 @@ def test_qknorm(batch_size: int, n_k: int, n_q: int, head_dim: int) -> None: triton.testing.assert_close(q_k_aot[1], q_k_jit[1], atol=1e-2, rtol=1e-2) +@pytest.mark.parametrize( + "dtype,pad", [(torch.bfloat16, 0), (torch.float16, 0), (torch.bfloat16, 4)] +) +def test_two_rows_per_warp_match_one_row_per_warp(dtype: torch.dtype, pad: int) -> None: + """head_dim 128 runs two rows per warp; its bytes must equal the one-row-per-warp + kernel's. q and k slice one qkv row as Qwen3 passes them; 5 q heads put a q/k + boundary inside a row pair, the odd row count leaves the last warp one row, and + 4099 tokens need more than one pass of the persistent grid on H100 and B200. + pad=4 leaves odd tokens only 8 B aligned, so the launcher must keep one row per warp.""" + from sglang.kernels.ops.layernorm.norm import _jit_qknorm_module + + T, Hq, Hk, D = 4099, 5, 2, 128 + g = torch.Generator(device="cpu").manual_seed(0) + qkv = (3 * torch.randn(T, (Hq + 2 * Hk) * D + pad, generator=g)).to(DEVICE, dtype) + qw = (torch.rand(D, generator=g) + 0.5).to(DEVICE, dtype) + kw = (torch.rand(D, generator=g) + 0.5).to(DEVICE, dtype) + outputs = [] + for half_warp in (False, True): + out = qkv.clone() + q = out[:, : Hq * D].view(T, Hq, D) + k = out[:, Hq * D : (Hq + Hk) * D].view(T, Hk, D) + _jit_qknorm_module(D, dtype, half_warp=half_warp).qknorm(q, k, qw, kw, 1e-6) + outputs.append(out) + assert torch.equal(outputs[0], outputs[1]) + + if __name__ == "__main__": sys.exit(pytest.main([__file__, "-v", "-s"]))