[Perf] qknorm: run head_dim-128 rows two per warp with 16B lanes, bit-identical (#45214)
Signed-off-by: Monishver Chandrasekaran <monishverchandrasekaran@gmail.com> Co-authored-by: Xiaoyu Zhang <1182563586@qq.com>
This commit is contained in:
co-authored by
Xiaoyu Zhang
parent
81841d6bdc
commit
178febef5d
@@ -67,6 +67,69 @@ __global__ void fused_qknorm_warp(const QKNormParams __grid_constant__ params) {
|
||||
PDLTriggerSecondary<kUsePDL>(); // launch secondary kernel
|
||||
}
|
||||
|
||||
// Head_dim-128 variant of fused_qknorm_warp: two rows per warp, 16 lanes x 16 B per row.
|
||||
// Each lane replays the warp kernel's butterfly on two partial sums, so the output is bit-identical.
|
||||
template <bool kUsePDL, typename Float>
|
||||
__global__ void fused_qknorm_half_warp(const QKNormParams __grid_constant__ params) {
|
||||
using namespace device;
|
||||
|
||||
constexpr int64_t kHeadDim = 128;
|
||||
constexpr uint32_t kRowLanes = 16;
|
||||
using Packed = packed_t<Float>;
|
||||
constexpr uint32_t kPacks = kHeadDim / (2 * kRowLanes);
|
||||
using Storage = AlignedVector<Packed, kPacks>;
|
||||
|
||||
static_assert(sizeof(Float) == 2, "Only support FP16/BF16");
|
||||
const auto& [q, k, q_stride, k_stride, num_qo_heads, num_kv_heads, eps, q_weight, k_weight, num_tokens] = params;
|
||||
|
||||
const auto lane_id = threadIdx.x % kWarpThreads;
|
||||
const auto row_lane = lane_id % kRowLanes;
|
||||
const auto num_workers = gridDim.x * kWarpsPerBlock;
|
||||
const auto num_q_and_k_heads = num_qo_heads + num_kv_heads;
|
||||
const auto num_rows = num_q_and_k_heads * num_tokens;
|
||||
const auto num_pairs = (num_rows + 1) / 2;
|
||||
const auto start_worker_id = blockIdx.x * kWarpsPerBlock + threadIdx.x / kWarpThreads;
|
||||
|
||||
PDLWaitPrimary<kUsePDL>(); // wait for primary kernel
|
||||
|
||||
for (auto pair = start_worker_id; pair < num_pairs; pair += num_workers) {
|
||||
// a missing second row recomputes the last one and skips the store
|
||||
const auto wanted_row = 2 * pair + lane_id / kRowLanes;
|
||||
const auto row = wanted_row < num_rows ? wanted_row : num_rows - 1;
|
||||
const int64_t token_id = row / num_q_and_k_heads;
|
||||
const int64_t head_id = row % num_q_and_k_heads;
|
||||
const auto load_q = head_id < num_qo_heads;
|
||||
const auto input = load_q ? pointer::offset(q, 2 * (token_id * q_stride + head_id * kHeadDim))
|
||||
: pointer::offset(k, 2 * (token_id * k_stride + head_id * kHeadDim));
|
||||
const auto weight = load_q ? q_weight : k_weight;
|
||||
auto vec = load_as<Storage>(input, row_lane);
|
||||
const auto weight_vec = load_as<Storage>(weight, row_lane);
|
||||
|
||||
// part[h] is the sum of warp-kernel lane 2 * row_lane + h, and reduce_sum below
|
||||
// is that kernel's 16..2 xor steps; its final xor-1 step adds the pair.
|
||||
float part[2] = {0.0f, 0.0f};
|
||||
#pragma unroll
|
||||
for (auto i = 0u; i < kPacks; ++i) {
|
||||
const auto x = cast<fp32x2_t>(vec[i]);
|
||||
part[i / 2] += x.x * x.x;
|
||||
part[i / 2] += x.y * x.y;
|
||||
}
|
||||
part[0] = warp::reduce_sum<kRowLanes>(part[0]);
|
||||
part[1] = warp::reduce_sum<kRowLanes>(part[1]);
|
||||
const float norm_factor = math::rsqrt((part[0] + part[1]) / kHeadDim + eps);
|
||||
|
||||
#pragma unroll
|
||||
for (auto i = 0u; i < kPacks; ++i) {
|
||||
const auto x = cast<fp32x2_t>(vec[i]);
|
||||
const auto w = cast<fp32x2_t>(weight_vec[i]);
|
||||
vec[i] = cast<Packed, fp32x2_t>({x.x * norm_factor * w.x, x.y * norm_factor * w.y});
|
||||
}
|
||||
if (wanted_row < num_rows) store_as<Storage>(input, vec, row_lane);
|
||||
}
|
||||
|
||||
PDLTriggerSecondary<kUsePDL>(); // launch secondary kernel
|
||||
}
|
||||
|
||||
// For CTA level, used for head_dim > 256 (512,1024)
|
||||
template <int64_t kHeadDim, bool kUsePDL, typename Float>
|
||||
__global__ void fused_qknorm_cta(const QKNormParams __grid_constant__ params) {
|
||||
@@ -102,12 +165,23 @@ __global__ void fused_qknorm_cta(const QKNormParams __grid_constant__ params) {
|
||||
PDLTriggerSecondary<kUsePDL>(); // launch secondary kernel
|
||||
}
|
||||
|
||||
// Head_dims that fused_qknorm_half_warp covers
|
||||
template <int64_t kHeadDim>
|
||||
inline constexpr bool kUseHalfWarp =
|
||||
#ifdef USE_ROCM
|
||||
false;
|
||||
#else
|
||||
kHeadDim == 128;
|
||||
#endif
|
||||
|
||||
// Warp-level kernel struct for head_dim <= 256
|
||||
template <int64_t kHeadDim, bool kUsePDL, typename DType>
|
||||
template <int64_t kHeadDim, bool kUsePDL, typename DType, bool kAllowHalfWarp = true>
|
||||
struct QKNormKernelWarp {
|
||||
static_assert(std::is_same_v<DType, fp16_t> || std::is_same_v<DType, bf16_t>);
|
||||
static_assert(!host::norm::should_use_cta<DType, kHeadDim>(), "Use QKNormKernelCTA for head_dim > 256");
|
||||
static constexpr auto kernel = fused_qknorm_warp<kHeadDim, kUsePDL, DType>;
|
||||
// kAllowHalfWarp = false keeps the one-row-per-warp kernel, which tests compare against
|
||||
static constexpr bool kHalfWarp = kAllowHalfWarp && kUseHalfWarp<kHeadDim>;
|
||||
|
||||
static void
|
||||
run(const tvm::ffi::TensorView q,
|
||||
@@ -161,17 +235,33 @@ struct QKNormKernelWarp {
|
||||
.num_tokens = num_tokens,
|
||||
};
|
||||
|
||||
static const uint32_t max_occupancy = runtime::get_blocks_per_sm(kernel, kThreadsPerBlock);
|
||||
static const uint32_t kNumSM = runtime::get_sm_count(device.unwrap().device_id);
|
||||
const auto num_rows = (num_qo_heads + num_kv_heads) * num_tokens;
|
||||
if constexpr (kHalfWarp) {
|
||||
// the half-warp kernel moves 16 B per lane, the warp kernel only 8 B
|
||||
const auto address_bits = reinterpret_cast<uintptr_t>(q.data_ptr()) | reinterpret_cast<uintptr_t>(k.data_ptr()) |
|
||||
reinterpret_cast<uintptr_t>(q_weight.data_ptr()) |
|
||||
reinterpret_cast<uintptr_t>(k_weight.data_ptr()) |
|
||||
static_cast<uintptr_t>(2 * (Sq.unwrap() | Sk.unwrap()));
|
||||
if (address_bits % 16 == 0) {
|
||||
return launch<fused_qknorm_half_warp<kUsePDL, DType>>(params, div_ceil(num_rows, 2u), device.unwrap());
|
||||
}
|
||||
}
|
||||
launch<kernel>(params, num_rows, device.unwrap());
|
||||
}
|
||||
|
||||
template <auto kKernel>
|
||||
static void launch(const QKNormParams& params, uint32_t num_works, DLDevice device) {
|
||||
using namespace host;
|
||||
|
||||
static const uint32_t max_occupancy = runtime::get_blocks_per_sm(kKernel, kThreadsPerBlock);
|
||||
static const uint32_t kNumSM = runtime::get_sm_count(device.device_id);
|
||||
|
||||
// choose kernel based on dtype
|
||||
const auto num_works = (num_qo_heads + num_kv_heads) * num_tokens;
|
||||
const auto needed_blocks = div_ceil(num_works, kWarpsPerBlock);
|
||||
|
||||
// we use persistent kernel, which limit the number of blocks to reduce overhead
|
||||
const auto num_blocks = std::min(kNumSM * max_occupancy, needed_blocks);
|
||||
LaunchKernel(num_blocks, kThreadsPerBlock, device.unwrap()) //
|
||||
.enable_pdl(kUsePDL)(kernel, params);
|
||||
LaunchKernel(num_blocks, kThreadsPerBlock, device) //
|
||||
.enable_pdl(kUsePDL)(kKernel, params);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -248,10 +338,10 @@ struct QKNormKernelCTA {
|
||||
};
|
||||
|
||||
// Unified dispatch: select warp or CTA kernel based on head_dim
|
||||
template <int64_t kHeadDim, bool kUsePDL, typename DType>
|
||||
template <int64_t kHeadDim, bool kUsePDL, typename DType, bool kAllowHalfWarp = true>
|
||||
using QKNormKernel = std::conditional_t<
|
||||
host::norm::should_use_cta<DType, kHeadDim>(),
|
||||
QKNormKernelCTA<kHeadDim, kUsePDL, DType>,
|
||||
QKNormKernelWarp<kHeadDim, kUsePDL, DType>>;
|
||||
QKNormKernelWarp<kHeadDim, kUsePDL, DType, kAllowHalfWarp>>;
|
||||
|
||||
} // namespace sglang
|
||||
|
||||
@@ -21,8 +21,12 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@cache_once
|
||||
def _jit_qknorm_module(head_dim: int, dtype: torch.dtype) -> Module:
|
||||
args = make_cpp_args(head_dim, is_arch_support_pdl(), dtype)
|
||||
def _jit_qknorm_module(
|
||||
head_dim: int, dtype: torch.dtype, half_warp: bool = True
|
||||
) -> Module:
|
||||
"""``half_warp=False`` keeps the one-row-per-warp kernel where the two-rows
|
||||
variant would apply; tests use it as the bit-exactness oracle."""
|
||||
args = make_cpp_args(head_dim, is_arch_support_pdl(), dtype, half_warp)
|
||||
return load_jit(
|
||||
"qknorm",
|
||||
*args,
|
||||
|
||||
@@ -82,5 +82,31 @@ def test_qknorm(batch_size: int, n_k: int, n_q: int, head_dim: int) -> None:
|
||||
triton.testing.assert_close(q_k_aot[1], q_k_jit[1], atol=1e-2, rtol=1e-2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"dtype,pad", [(torch.bfloat16, 0), (torch.float16, 0), (torch.bfloat16, 4)]
|
||||
)
|
||||
def test_two_rows_per_warp_match_one_row_per_warp(dtype: torch.dtype, pad: int) -> None:
|
||||
"""head_dim 128 runs two rows per warp; its bytes must equal the one-row-per-warp
|
||||
kernel's. q and k slice one qkv row as Qwen3 passes them; 5 q heads put a q/k
|
||||
boundary inside a row pair, the odd row count leaves the last warp one row, and
|
||||
4099 tokens need more than one pass of the persistent grid on H100 and B200.
|
||||
pad=4 leaves odd tokens only 8 B aligned, so the launcher must keep one row per warp."""
|
||||
from sglang.kernels.ops.layernorm.norm import _jit_qknorm_module
|
||||
|
||||
T, Hq, Hk, D = 4099, 5, 2, 128
|
||||
g = torch.Generator(device="cpu").manual_seed(0)
|
||||
qkv = (3 * torch.randn(T, (Hq + 2 * Hk) * D + pad, generator=g)).to(DEVICE, dtype)
|
||||
qw = (torch.rand(D, generator=g) + 0.5).to(DEVICE, dtype)
|
||||
kw = (torch.rand(D, generator=g) + 0.5).to(DEVICE, dtype)
|
||||
outputs = []
|
||||
for half_warp in (False, True):
|
||||
out = qkv.clone()
|
||||
q = out[:, : Hq * D].view(T, Hq, D)
|
||||
k = out[:, Hq * D : (Hq + Hk) * D].view(T, Hk, D)
|
||||
_jit_qknorm_module(D, dtype, half_warp=half_warp).qknorm(q, k, qw, kw, 1e-6)
|
||||
outputs.append(out)
|
||||
assert torch.equal(outputs[0], outputs[1])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(pytest.main([__file__, "-v", "-s"]))
|
||||
|
||||
Reference in New Issue
Block a user