diff --git a/python/sglang/kernels/jit/csrc/deepselect/README.md b/python/sglang/kernels/jit/csrc/deepselect/README.md new file mode 100644 index 0000000000..14a46c612f --- /dev/null +++ b/python/sglang/kernels/jit/csrc/deepselect/README.md @@ -0,0 +1,59 @@ +# DeepSelect top-k + +`vendor/` is [deepseek-ai/DeepSelect](https://github.com/deepseek-ai/DeepSelect) +(commit `382d62a`, plus the SM90 cluster variant from upstream PR #14), stripped +to the kernel implementation. MIT licensed; see `vendor/LICENSE`. + +`entry.cuh` is the SGLang side: tensor validation, the `TopkSelectArgs` block, +and the two host entry points the JIT exports. `python/sglang/kernels/ops/deep_select.py` +is the wrapper that compiles and calls them. + +## What is vendored + +| Path under `vendor/` | What it is | +| --- | --- | +| `structs.h` | `TopkSelectArgs` -- the launcher's plain-C argument block, plus the stride-alignment constants | +| `cuda_kernels/config.h` | `TopkSelectConfig<...>` -- every compile-time knob of a kernel instance | +| `cuda_kernels/{utils,bit_utils}.cuh` | small device helpers | +| `cuda_kernels/common_parts.cuh` | the shared kernel body (TMA pipeline, radix pass, epilogue) | +| `cuda_kernels/v3/topk_select.cuh` | bf16, one CTA per row | +| `cuda_kernels/v3_cluster/topk_select.cuh` | bf16, one cluster per row (8 CTAs on SM90, 16 on SM100/SM103) | +| `cuda_kernels/v3_fp32/topk_select.cuh` | fp32 | +| `3rdparty/kerutils/` | upstream's header-only helper library, subset | + +Each `topk_select.cuh` defines one kernel plus its launcher, +`run_topk_select_kernel(const TopkSelectArgs&)`, in its own namespace +(`topk_select_bf16_normal`, `topk_select_bf16_cluster`, `topk_select_fp32`). + +Dropped from the vendored tree, because a JIT build does not need them: + +- `cuda_kernels/*/instantiations/**` -- 86 explicit instantiation `.cu` files. + They exist so an AOT build can compile every configuration into one shared + object. `entry.cuh` instantiates the one or two a module can reach. +- `api.cpp` -- the torch/pybind entry point and the runtime dispatch that + picked a configuration from `(dtype, topk, vocab_size, batch_size, arch)`. + That decision is now split between the Python module factory and `entry.cuh`. +- `dispatch_utils.h` -- `BOOL_SWITCH` / `INTEGER_TYPE_SWITCH`, which include + `torch/extension.h` and exist only to expand that dispatch. +- `cuda_kernels/*/topk_select.h` -- forward declarations of + `run_topk_select_kernel`, needed only to split the instantiations into + separate translation units. + +## Build inputs + +Include paths: `vendor/` (the `#include "cuda_kernels/..."` lines resolve +against it) and `vendor/3rdparty/kerutils/include`. CUTLASS supplies +`cute/arch/copy_sm90_tma.hpp` and `cutlass/arch/barrier.h`, and is a registered +JIT dependency (`extra_dependencies=["cutlass"]`). + +Beyond the JIT defaults the kernels need `--expt-extended-lambda` and +`--ftz=false`; the latter is load-bearing and must follow `--use_fast_math`, +because the radix pass simulates integer counters with fp32 addition and needs +subnormals. + +## Re-syncing with upstream + +Copy the files listed above into `vendor/`. They are unmodified apart from +trailing whitespace and the removed `#include "topk_select.h"` line at the top +of each `topk_select.cuh`. `vendor/.clang-format` disables formatting so a +re-sync stays a plain copy. diff --git a/python/sglang/kernels/jit/csrc/deepselect/entry.cuh b/python/sglang/kernels/jit/csrc/deepselect/entry.cuh new file mode 100644 index 0000000000..9003eea300 --- /dev/null +++ b/python/sglang/kernels/jit/csrc/deepselect/entry.cuh @@ -0,0 +1,294 @@ +#pragma once + +// The vendored DeepSelect sources declare everything at global scope, so they +// are included before `namespace sglang` is opened. +#include +#include + +#include +#include + +#include +#include + +#include "vendor/cuda_kernels/v3/topk_select.cuh" +#include "vendor/cuda_kernels/v3_cluster/topk_select.cuh" +#include "vendor/cuda_kernels/v3_fp32/topk_select.cuh" +#include +#include +#include + +namespace sglang { + +/** + * \brief Host entry points for the vendored DeepSelect top-k kernels. + * + * Upstream resolves a `TopkSelectConfig` from the call at runtime, expanding + * every boolean and index-type axis through nested `BOOL_SWITCH` macros. Here + * those axes -- value dtype, index dtype, `sorted_value`, `sorted_index`, + * `return_value`, the top-k bucket and the cluster size -- are template + * arguments picked by the Python module factory, which leaves exactly one + * decision for C++: which of a bucket's two tunings the launch shape wants. + */ +namespace deepselect { + +namespace details { + +/// The two families of one-CTA-per-row kernels live in separate namespaces. +template +inline auto run_normal(const TopkSelectArgs& args) -> void { + if constexpr (std::is_same_v) { + topk_select_fp32::run_topk_select_kernel(args); + } else { + topk_select_bf16_normal::run_topk_select_kernel(args); + } +} + +/// Validated launch arguments plus the device they were validated against. +struct LaunchPlan { + TopkSelectArgs args; + DLDevice device; +}; + +/** + * \brief Validate the tensors and build the argument block for one launch. + * + * \tparam Config The specialisation the caller is about to run. Its + * `sorted_value` / `sorted_index` / `return_value` become the runtime + * flags, which the kernel launcher then asserts back against them. + */ +template +auto make_plan( + tvm::ffi::TensorView input, + tvm::ffi::Optional output_value, + tvm::ffi::TensorView output_index, + tvm::ffi::TensorView kernel_output_index, + tvm::ffi::Optional begin, + tvm::ffi::Optional end, + tvm::ffi::Optional output_index_offset, + int64_t topk, + int64_t input_storage_bytes, + int64_t idx_oob_fill_value, + double value_oob_fill_value, + bool abort_when_nan_found) -> LaunchPlan { + using namespace host; + using ValueT = typename Config::ValueT; + using OutIdxT = typename Config::OutIdxT; + + auto batch = SymbolicSize{"batch_size"}; + auto vocab = SymbolicSize{"vocab_size"}; + auto input_stride = SymbolicSize{"input_row_stride"}; + auto index_stride = SymbolicSize{"index_row_stride"}; + auto kernel_index_stride = SymbolicSize{"kernel_index_row_stride"}; + auto value_stride = SymbolicSize{"value_row_stride"}; + auto device = SymbolicDevice{}; + + CHECK_HOST(topk > 0 && topk <= static_cast(Config::max_topk)) + << "topk must be in (0, " << Config::max_topk << "] for this module, got " << topk; + CHECK_HOST(!begin.has_value()) << "`begin` is not supported currently"; + CHECK_HOST( + idx_oob_fill_value >= std::numeric_limits::min() && + idx_oob_fill_value <= std::numeric_limits::max()) + << "idx_oob_fill_value must fit in int32"; + CHECK_HOST(input_storage_bytes >= 0) << "input_storage_bytes must be non-negative"; + + TensorMatcher({batch, vocab}) + .with_strides({input_stride, 1}) + .with_device(device) + .with_dtype() + .verify(input); + TensorMatcher({batch, topk}) + .with_strides({index_stride, 1}) + .with_device(device) + .with_dtype() + .verify(output_index); + TensorMatcher({batch, topk}) + .with_strides({kernel_index_stride, 1}) + .with_device(device) + .with_dtype() + .ensure_alignment(OUTPUT_STRIDE_ALIGNMENT_REQUIREMENT) + .verify(kernel_output_index); + + void* output_value_ptr = nullptr; + uint64_t stride_output_value = 0; + if constexpr (Config::return_value) { + CHECK_HOST(output_value.has_value()) << "output_value is required when return_value is enabled"; + TensorMatcher({batch, topk}) + .with_strides({value_stride, 1}) + .with_device(device) + .with_dtype() + .ensure_alignment(OUTPUT_STRIDE_ALIGNMENT_REQUIREMENT) + .verify(output_value.value()); + output_value_ptr = output_value.value().data_ptr(); + stride_output_value = static_cast(value_stride.unwrap()); + } else { + CHECK_HOST(!output_value.has_value()) << "output_value must be absent when return_value is disabled"; + } + + int* end_ptr = nullptr; + if (end.has_value()) { + TensorMatcher({batch}).with_device(device).with_dtype().verify(end.value()); + end_ptr = static_cast(end.value().data_ptr()); + } + int* index_offset_ptr = nullptr; + if (output_index_offset.has_value()) { + TensorMatcher({batch}).with_device(device).with_dtype().verify(output_index_offset.value()); + index_offset_ptr = static_cast(output_index_offset.value().data_ptr()); + } + + const auto batch_size = static_cast(batch.unwrap()); + const auto vocab_size = static_cast(vocab.unwrap()); + CHECK_HOST(vocab_size < MAX_VOCAB_SIZE) << "vocab_size must be < " << MAX_VOCAB_SIZE << ", got " << vocab_size; + + const DLDevice dev = device.unwrap(); + const auto input_stride_elements = input_stride.unwrap(); + CHECK_HOST(input_stride_elements >= 0) << "input row stride must be non-negative"; + CHECK_HOST(static_cast(input_stride_elements) * sizeof(ValueT) % INPUT_STRIDE_ALIGNMENT_REQUIREMENT == 0) + << "input row stride must be a multiple of " << INPUT_STRIDE_ALIGNMENT_REQUIREMENT << " bytes"; + CHECK_HOST(static_cast(index_stride.unwrap()) * sizeof(OutIdxT) % OUTPUT_STRIDE_ALIGNMENT_REQUIREMENT == 0) + << "output_index row stride must be a multiple of " << OUTPUT_STRIDE_ALIGNMENT_REQUIREMENT << " bytes"; + + if (batch_size != 0 && vocab_size != 0) { + constexpr uint64_t address_alignment = SGL_CUDA_ARCH == 900 ? 16 : 32; + CHECK_HOST(reinterpret_cast(input.data_ptr()) % address_alignment == 0) + << "input address must be aligned to " << address_alignment << " bytes"; + const uint64_t row_bytes = static_cast(vocab_size) * sizeof(ValueT); + const uint64_t padded_row_bytes = (row_bytes + 127) / 128 * 128; + const uint64_t required_bytes = + static_cast(batch_size - 1) * input_stride_elements * sizeof(ValueT) + padded_row_bytes; + CHECK_HOST(required_bytes <= static_cast(input_storage_bytes)) + << "input storage must include the final row padded to 128 bytes"; + } + + return LaunchPlan{ + TopkSelectArgs{ + batch_size, + vocab_size, + static_cast(topk), + input.data_ptr(), + output_value_ptr, + kernel_output_index.data_ptr(), + nullptr, // `begin` is unimplemented upstream + end_ptr, + index_offset_ptr, + static_cast(input_stride.unwrap()), + stride_output_value, + static_cast(kernel_index_stride.unwrap()), + Config::sorted_value, + Config::sorted_index, + Config::return_value, + static_cast(idx_oob_fill_value), + static_cast(value_oob_fill_value), + abort_when_nan_found, + host::runtime::get_max_smem_per_block(dev.device_id), + host::LaunchKernel::resolve_device(dev), + }, + dev, + }; +} + +} // namespace details + +/** + * \brief Top-k through the one-CTA-per-row kernels. + * + * \tparam ConfigWave1 Tuning for a batch that fits in a single wave. + * \tparam ConfigWaves Tuning for anything larger. A bucket with no wave split + * leaves it defaulted; one kernel is then instantiated and the SM-count + * query drops out. + */ +template +struct TopkNormal { + static_assert(ConfigWave1::cluster_size == 1 && ConfigWaves::cluster_size == 1); + static_assert(std::is_same_v); + static_assert(std::is_same_v); + static_assert(ConfigWave1::max_topk == ConfigWaves::max_topk); + static_assert(ConfigWave1::sorted_value == ConfigWaves::sorted_value); + static_assert(ConfigWave1::sorted_index == ConfigWaves::sorted_index); + static_assert(ConfigWave1::return_value == ConfigWaves::return_value); + + static auto topk( + tvm::ffi::TensorView input, + tvm::ffi::Optional output_value, + tvm::ffi::TensorView output_index, + tvm::ffi::TensorView kernel_output_index, + tvm::ffi::Optional begin, + tvm::ffi::Optional end, + tvm::ffi::Optional output_index_offset, + int64_t topk, + int64_t input_storage_bytes, + int64_t idx_oob_fill_value, + double value_oob_fill_value, + bool abort_when_nan_found) -> void { + const auto plan = details::make_plan( + input, + output_value, + output_index, + kernel_output_index, + begin, + end, + output_index_offset, + topk, + input_storage_bytes, + idx_oob_fill_value, + value_oob_fill_value, + abort_when_nan_found); + if (plan.args.batch_size == 0) return; + if constexpr (std::is_same_v) { + details::run_normal(plan.args); + } else { + const auto num_sm = host::runtime::get_sm_count(plan.device.device_id); + if (plan.args.batch_size <= num_sm) { + details::run_normal(plan.args); + } else { + details::run_normal(plan.args); + } + } + } +}; + +/** + * \brief Top-k through the cluster kernel: `Config::cluster_size` CTAs per row. + * + * Whether a launch belongs here -- small batch, long rows, bf16 -- is decided + * in Python, which also picks the cluster size: 8 CTAs on SM90, 16 on + * SM100/SM103, where 16 needs the non-portable cluster attribute. + */ +template +struct TopkCluster { + static_assert(Config::cluster_size > 1); + + static auto topk( + tvm::ffi::TensorView input, + tvm::ffi::Optional output_value, + tvm::ffi::TensorView output_index, + tvm::ffi::TensorView kernel_output_index, + tvm::ffi::Optional begin, + tvm::ffi::Optional end, + tvm::ffi::Optional output_index_offset, + int64_t topk, + int64_t input_storage_bytes, + int64_t idx_oob_fill_value, + double value_oob_fill_value, + bool abort_when_nan_found) -> void { + const auto plan = details::make_plan( + input, + output_value, + output_index, + kernel_output_index, + begin, + end, + output_index_offset, + topk, + input_storage_bytes, + idx_oob_fill_value, + value_oob_fill_value, + abort_when_nan_found); + if (plan.args.batch_size == 0) return; + topk_select_bf16_cluster::run_topk_select_kernel(plan.args); + } +}; + +} // namespace deepselect + +} // namespace sglang diff --git a/python/sglang/kernels/jit/csrc/deepselect/vendor/.clang-format b/python/sglang/kernels/jit/csrc/deepselect/vendor/.clang-format new file mode 100644 index 0000000000..06bc3bf2e5 --- /dev/null +++ b/python/sglang/kernels/jit/csrc/deepselect/vendor/.clang-format @@ -0,0 +1,4 @@ +# Vendored from deepseek-ai/DeepSelect; keep the sources byte-comparable with +# upstream so re-syncing stays a plain copy. See README.md. +DisableFormat: true +SortIncludes: Never diff --git a/python/sglang/kernels/jit/csrc/deepselect/vendor/3rdparty/kerutils/README.md b/python/sglang/kernels/jit/csrc/deepselect/vendor/3rdparty/kerutils/README.md new file mode 100644 index 0000000000..a07ffb2882 --- /dev/null +++ b/python/sglang/kernels/jit/csrc/deepselect/vendor/3rdparty/kerutils/README.md @@ -0,0 +1,20 @@ +# Kerutils + +Kerutils is a library that provides: + +- Wrappers for low-level PTX instructions +- Helpers for checking tensor device / shape / stride / dtype, useful in kernel library's dispatcher + +> This copy is a subset of upstream Kerutils: the CuTe UTCMMA / 2SM TMA copy wrappers, +> the GeMM helpers and the Kernel Insight (KI) library have been removed because +> DeepSelect does not use them. + +## Getting Started + +To use Kerutils, simply clone this repo (or register it as a submodule, if you're working in a repo), and include ``. You should add `include/` into your compiler's `includePath` (for example, by adding `-Ipath/to/kerutils/include`). + +## Organization + +- `include/host` - Host-side helper functions +- `include/device` - Device-side PTX wrappers and helper functions, organized by GPU architectures +- `supplemental/` - Files in this directory are NOT included by `kerutils/kerutils.cuh` by default, to reduce compilation time. `supplemental/torch_tensors.h` provides various handy guard functions for examining a libTorch tensor. diff --git a/python/sglang/kernels/jit/csrc/deepselect/vendor/3rdparty/kerutils/include/kerutils/common/common.h b/python/sglang/kernels/jit/csrc/deepselect/vendor/3rdparty/kerutils/include/kerutils/common/common.h new file mode 100644 index 0000000000..923ca4fa2f --- /dev/null +++ b/python/sglang/kernels/jit/csrc/deepselect/vendor/3rdparty/kerutils/include/kerutils/common/common.h @@ -0,0 +1,15 @@ +#pragma once + +namespace kerutils {} + +#define KU_PRINTLN(fmt, ...) { cute::print(fmt, ##__VA_ARGS__); print("\n"); } + +namespace ku = kerutils; + +#ifdef __CUDACC__ +#define KERUTILS_IS_BUILD_ON_CUDA +#endif + +#ifndef KERUTILS_IS_BUILD_ON_CUDA +#error "KERUTILS_IS_BUILD_ON_CUDA must be defined. It is defined automatically when compiling with NVCC; pass `-DKERUTILS_IS_BUILD_ON_CUDA` when compiling with a host compiler." +#endif diff --git a/python/sglang/kernels/jit/csrc/deepselect/vendor/3rdparty/kerutils/include/kerutils/device/cuda/common.h b/python/sglang/kernels/jit/csrc/deepselect/vendor/3rdparty/kerutils/include/kerutils/device/cuda/common.h new file mode 100644 index 0000000000..05d671eea2 --- /dev/null +++ b/python/sglang/kernels/jit/csrc/deepselect/vendor/3rdparty/kerutils/include/kerutils/device/cuda/common.h @@ -0,0 +1,75 @@ +/* +Common data types and macros that are used across the kerutils library. +*/ +#pragma once + +#include +#include + +#include +#include +#include // For CUTE_DEVICE + +namespace kerutils { + +// Cache hints +enum class CacheHint { + EVICT_FIRST, + EVICT_NORMAL, + EVICT_LAST, + EVICT_UNCHANGED, + NO_ALLOCATE +}; + +// Prefetch size +enum class PrefetchSize { + B64, + B128, + B256 +}; + +using nvbf16 = __nv_bfloat16; +using nvbf16x2 = __nv_bfloat162; +using nve4m3 = __nv_fp8_e4m3; +using nve4m3x2 = __nv_fp8x2_e4m3; +using nve4m3x4 = __nv_fp8x4_e4m3; + +using bf16 = cutlass::bfloat16_t; +using transac_bar_t = cutlass::arch::ClusterTransactionBarrier; + +} + +#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 800)) +#define KERUTILS_ENABLE_SM80 +#elif (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ < 800)) +static_assert(false, "kerutils doesn't support SM architectures below SM80"); +#endif + +#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)) +#define KERUTILS_ENABLE_SM90 +#endif + +#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900 && __CUDA_ARCH__ < 1000)) +#define KERUTILS_ENABLE_SM90A +#endif + +#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000)) +#define KERUTILS_ENABLE_SM100 +#endif + +#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000 && __CUDA_ARCH__ < 1200)) +#define KERUTILS_ENABLE_SM100A +#endif + +#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1030 && __CUDA_ARCH__ < 1200)) +#define KERUTILS_ENABLE_SM103A +#endif + +#if (defined(__CLION_IDE__) || defined(__VSCODE_IDE__)) +#define KERUTILS_ENABLE_SM80 +#define KERUTILS_ENABLE_SM90 +#define KERUTILS_ENABLE_SM90A +#define KERUTILS_ENABLE_SM100 +#define KERUTILS_ENABLE_SM100A +#define KERUTILS_ENABLE_SM103A +#endif diff --git a/python/sglang/kernels/jit/csrc/deepselect/vendor/3rdparty/kerutils/include/kerutils/device/cuda/sm100/intrinsics.cuh b/python/sglang/kernels/jit/csrc/deepselect/vendor/3rdparty/kerutils/include/kerutils/device/cuda/sm100/intrinsics.cuh new file mode 100644 index 0000000000..a24c123e34 --- /dev/null +++ b/python/sglang/kernels/jit/csrc/deepselect/vendor/3rdparty/kerutils/include/kerutils/device/cuda/sm100/intrinsics.cuh @@ -0,0 +1,649 @@ +#pragma once + +#include + +#include "kerutils/device/cuda/common.h" + +namespace kerutils { + +// tma gather4 (https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-cp-async-bulk-tensor) +// Please pay attention that the coordinates of TMA gather4 are int32, which may lead to overflow under some scenarios +CUTE_DEVICE +void tma_gather4(const void* desc_ptr, transac_bar_t &mbar_ptr, void* smem_ptr, int col_idx, int4 row_idxs, int64_t cache_hint) { + uint32_t smem_addr = cute::cast_smem_ptr_to_uint(smem_ptr); + uint32_t mbar_addr = cute::cast_smem_ptr_to_uint(&mbar_ptr); + asm volatile( + "cp.async.bulk.tensor.2d.shared::cta.global.tile::gather4.mbarrier::complete_tx::bytes.cta_group::1.L2::cache_hint [%0], [%1, {%2, %3, %4, %5, %6}], [%7], %8;\n" + : + : "r"(smem_addr), "l"(desc_ptr), "r"(col_idx), + "r"(row_idxs.x), "r"(row_idxs.y), "r"(row_idxs.z), "r"(row_idxs.w), + "r"(mbar_addr), "l"(cache_hint) + : "memory" + ); +} + +// tma gather4 prefetch (https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-cp-async-bulk-prefetch-tensor) +// Please pay attention that the coordinates of TMA gather4 are int32, which may lead to overflow under some scenarios +CUTE_DEVICE +void tma_gather4_prefetch(const void* desc_ptr, int col_idx, int4 row_idxs, int64_t cache_hint) { + asm volatile( + "cp.async.bulk.prefetch.tensor.2d.L2.global.tile::gather4.L2::cache_hint [%0, {%1, %2, %3, %4, %5}], %6;\n" + : + : "l"(desc_ptr), "r"(col_idx), + "r"(row_idxs.x), "r"(row_idxs.y), "r"(row_idxs.z), "r"(row_idxs.w), + "l"(cache_hint) + ); +} + +// tma gather4 with cta_group::2, allowing for synchronization across CTAs within a pair of CTAs (https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-cp-async-bulk-tensor) +template +CUTE_DEVICE void tma_gather4_cta_group_2(const void* desc_ptr, transac_bar_t &mbar_ptr, void* smem_ptr, int col_idx, int4 row_idxs, int64_t cache_hint) { + uint32_t smem_addr = cute::cast_smem_ptr_to_uint(smem_ptr); + uint32_t mbar_addr = cute::cast_smem_ptr_to_uint(&mbar_ptr); + if constexpr (USE_CTA0_MBAR) { + mbar_addr &= cute::Sm100MmaPeerBitMask; + } + asm volatile( + "cp.async.bulk.tensor.2d.shared::cta.global.tile::gather4.mbarrier::complete_tx::bytes.cta_group::2.L2::cache_hint [%0], [%1, {%2, %3, %4, %5, %6}], [%7], %8;\n" + : + : "r"(smem_addr), "l"(desc_ptr), "r"(col_idx), + "r"(row_idxs.x), "r"(row_idxs.y), "r"(row_idxs.z), "r"(row_idxs.w), + "r"(mbar_addr), "l"(cache_hint) + : "memory" + ); +} + +// Vectorized addition for float32 (https://docs.nvidia.com/cuda/parallel-thread-execution/#floating-point-instructions-add) +CUTE_DEVICE +float2 float2_add(const float2 &a, const float2 &b) { + float2 c; + asm volatile( + "add.f32x2 %0, %1, %2;\n" + : "=l"(reinterpret_cast(c)) + : "l"(reinterpret_cast(a)), + "l"(reinterpret_cast(b)) + ); + return c; +} + +// Vectorized multiplication for float32 (https://docs.nvidia.com/cuda/parallel-thread-execution/#floating-point-instructions-mul) +CUTE_DEVICE +float2 float2_mul(const float2 &a, const float2 &b) { + float2 c; + asm volatile( + "mul.f32x2 %0, %1, %2;\n" + : "=l"(reinterpret_cast(c)) + : "l"(reinterpret_cast(a)), + "l"(reinterpret_cast(b))); + return c; +} + +// Vectorized fused addition-multiplication for float32 (https://docs.nvidia.com/cuda/parallel-thread-execution/#floating-point-instructions-fma) +CUTE_DEVICE +float2 float2_fma(const float2 &a, const float2 &b, const float2 &c) { + // return a*b+c + float2 d; + asm volatile( + "fma.rn.f32x2 %0, %1, %2, %3;\n" + : "=l"(reinterpret_cast(d)) + : "l"(reinterpret_cast(a)), + "l"(reinterpret_cast(b)), + "l"(reinterpret_cast(c))); + return d; +} + +// Vectorized negation for foat32 +CUTE_DEVICE +float2 float2_neg(const float2 &a) { + float2 t = {-1.0f, -1.0f}; + return float2_mul(a, t); +} + +// st.bulk (https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-st-bulk) +CUTE_DEVICE +void st_bulk(void* dst_ptr, int64_t size) { + uint32_t dst_addr = cute::cast_smem_ptr_to_uint(dst_ptr); + asm volatile ( + "st.bulk.weak.shared::cta [%0], %1, 0;\n" + : + : "r"(dst_addr), "l"(size) + : "memory" + ); +} + +struct CUTE_ALIGNAS(16) CLCResponseObj { + // An opaque 16B value + char opaque[16]; +}; + +struct CLCResult { + int is_valid; + int x, y, z; +}; + +// Issue a CLC try_cancel query (https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-try-cancel) +CUTE_DEVICE +void issue_clc_query(transac_bar_t &bar, CLCResponseObj &response_obj) { + uint32_t response_addr = cute::cast_smem_ptr_to_uint(response_obj.opaque); + uint32_t mbarrier_addr = cute::cast_smem_ptr_to_uint(&bar); + asm volatile( + "clusterlaunchcontrol.try_cancel.async.shared::cta.mbarrier::complete_tx::bytes.b128 [%0], [%1];\n" + : + : "r"(response_addr), "r"(mbarrier_addr) + ); +} + +// Issue a CLC try_cancel query with .multicast::cluster::all (https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-try-cancel) +CUTE_DEVICE +void issue_clc_query_multicast_cluster_all(transac_bar_t &bar, CLCResponseObj &response_obj) { + uint32_t response_addr = cute::cast_smem_ptr_to_uint(response_obj.opaque); + uint32_t mbarrier_addr = cute::cast_smem_ptr_to_uint(&bar); + asm volatile( + "clusterlaunchcontrol.try_cancel.async.shared::cta.mbarrier::complete_tx::bytes.multicast::cluster::all.b128 [%0], [%1];\n" + : + : "r"(response_addr), "r"(mbarrier_addr) + ); +} + +// Get the result of a CLC query (https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-query-cancel) +// In this function, we separate get_first_ctaid::x/y/z and hope PTXAS's dead code elimination can remove unnecessary instructions +template +CUTE_DEVICE +CLCResult get_clc_query_response(CLCResponseObj &response_obj) { + uint32_t response_addr = cute::cast_smem_ptr_to_uint(&response_obj); + CLCResult result; + #define EMIT_ASM(LD_MODIFIER) \ + asm volatile( \ + "{\n" \ + ".reg .pred p1;\n\t" \ + ".reg .b128 clc_result;\n\t" \ + "ld" LD_MODIFIER ".shared.b128 clc_result, [%4];\n\t" \ + "clusterlaunchcontrol.query_cancel.is_canceled.pred.b128 p1, clc_result;\n\t" \ + "selp.u32 %3, 1, 0, p1;\n\t" \ + "@p1 clusterlaunchcontrol.query_cancel.get_first_ctaid::x.b32.b128 %0, clc_result;\n\t" \ + "@p1 clusterlaunchcontrol.query_cancel.get_first_ctaid::y.b32.b128 %1, clc_result;\n\t" \ + "@p1 clusterlaunchcontrol.query_cancel.get_first_ctaid::z.b32.b128 %2, clc_result;\n\t" \ + "}\n" \ + : "=r"(result.x), "=r"(result.y), "=r"(result.z), "=r"(result.is_valid) \ + : "r"(response_addr) \ + : "memory" \ + ); + if constexpr (USE_LD_ACQUIRE) { + EMIT_ASM(".acquire.cta"); + } else { + EMIT_ASM(""); + } + return result; +} + +// LDG.256 or LDG.256 with non-coherent cache (https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-ld) +// We use macro instead of function here, since we need a multi-level recursive dispatch based on template parameters if using function +// NC_STR should be either "" or ".nc" +// L1_CACHE_HINT_STR should be either "evict_first", "evict_normal", "evict_last", "evict_unchanged", or "no_allocate" +// L2_CACHE_HINT_STR should be either "evict_first", "evict_normal", or "evict_last" +// L2_PREFETCH_SIZE_STR should be either "64B", "128B", or "256B" +#define KU_LDG_256(global_addr, result, NC_STR, L1_CACHE_HINT_STR, L2_CACHE_HINT_STR, L2_PREFETCH_SIZE_STR) \ + { \ + static_assert(std::is_pointer_v || std::is_array_v, "`global_addr` must be a pointer"); \ + static_assert(std::is_pointer_v || std::is_array_v, "`result` must be a pointer"); \ + uint64_t* result_as_uint64_ptr = (uint64_t*)(result); \ + asm volatile( \ + "ld.global" NC_STR ".L1::" L1_CACHE_HINT_STR ".L2::" L2_CACHE_HINT_STR ".L2::" L2_PREFETCH_SIZE_STR ".v4.u64 {%0, %1, %2, %3}, [%4];\n" \ + : "=l"(result_as_uint64_ptr[0]), "=l"(result_as_uint64_ptr[1]), \ + "=l"(result_as_uint64_ptr[2]), "=l"(result_as_uint64_ptr[3]) \ + : "l"(global_addr) \ + ); \ + } + +// STG.256 (https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-st) +// L1_CACHE_HINT_STR should be either "evict_first", "evict_normal", "evict_last", "evict_unchanged", or "no_allocate" +// L2_CACHE_HINT_STR should be either "evict_first", "evict_normal", or "evict_last" +#define KU_STG_256(global_addr, src, L1_CACHE_HINT_STR, L2_CACHE_HINT_STR) \ + { \ + static_assert(std::is_pointer_v || std::is_array_v, "`global_addr` must be a pointer"); \ + static_assert(std::is_pointer_v || std::is_array_v, "`src` must be a pointer"); \ + uint64_t const* src_as_uint64_ptr = (uint64_t const*)(src); \ + asm volatile( \ + "st.global.L1::" L1_CACHE_HINT_STR ".L2::" L2_CACHE_HINT_STR ".v4.u64 [%0], {%1, %2, %3, %4};\n" \ + : \ + : "l"(global_addr), "l"(src_as_uint64_ptr[0]), "l"(src_as_uint64_ptr[1]), \ + "l"(src_as_uint64_ptr[2]), "l"(src_as_uint64_ptr[3]) \ + ); \ + } + +} + +namespace kerutils { + +// tcgen05.commit.cta_group::1 (https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen-async-sync-operations-commit) +CUTE_DEVICE +void umma_arrive_noelect(transac_bar_t &bar) { + uint32_t bar_intptr = cute::cast_smem_ptr_to_uint(&bar); + asm volatile( + "tcgen05.commit.cta_group::1.mbarrier::arrive::one.shared::cluster.b64 [%0];\n" + : + :"r"(bar_intptr) + ); +} + +// tcgen05.commit.cta_group::1, with multicast (https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen-async-sync-operations-commit) +CUTE_DEVICE +void umma_arrive_multicast_noelect(transac_bar_t &bar, uint16_t cta_mask) { + uint32_t bar_intptr = cute::cast_smem_ptr_to_uint(&bar); + asm volatile( + "tcgen05.commit.cta_group::1.mbarrier::arrive::one.shared::cluster.multicast::cluster.b64 [%0], %1;\n" + : + :"r"(bar_intptr), "h"(cta_mask) + ); +} + +// tcgen05.commit.cta_group::2 (https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen-async-sync-operations-commit) +CUTE_DEVICE +void umma_arrive_2x1SM_noelect(transac_bar_t &bar) { + uint32_t bar_intptr = cute::cast_smem_ptr_to_uint(&bar); + asm volatile( + "tcgen05.commit.cta_group::2.mbarrier::arrive::one.shared::cluster.b64 [%0];\n" + : + :"r"(bar_intptr) + ); +} + +// tcgen05.commit.cta_group::2, with multicast (https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen-async-sync-operations-commit) +CUTE_DEVICE +void umma_arrive_multicast_2x1SM_noelect(transac_bar_t &bar, uint16_t cta_mask) { + uint32_t bar_intptr = cute::cast_smem_ptr_to_uint(&bar); + asm volatile( + "tcgen05.commit.cta_group::2.mbarrier::arrive::one.shared::cluster.multicast::cluster.b64 [%0], %1;\n" + : + :"r"(bar_intptr), "h"(cta_mask) + ); +} + +// tcgen05.fence::before_thread_sync (https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-special-sync-operations-fence) +__device__ __forceinline__ void tcgen05_before_thread_sync() { + asm volatile("tcgen05.fence::before_thread_sync;"); +} + +// tcgen05.fence::after_thread_sync (https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-special-sync-operations-fence) +__device__ __forceinline__ void tcgen05_after_thread_sync() { + asm volatile("tcgen05.fence::after_thread_sync;"); +} + + +// Load from tensor memory, 32 data path lanes, 32-bit pattern, repeated N times. (https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-instructions-tcgen05-ld) +template +__device__ __forceinline__ +void tmem_ld_32dp32bNx(uint32_t tmem_start, void* data_) { + uint32_t* data = (uint32_t*)data_; + static_assert(kNumElements == 1 || kNumElements == 2 || kNumElements == 4 || kNumElements == 8 || kNumElements == 16 || kNumElements == 32 || kNumElements == 64 || kNumElements == 128, "Invalid kNumElements"); + // NOTE The following code crashes VSCode intellisense engine, so we disable it +#ifndef __VSCODE_IDE__ + [&](cute::index_sequence) { + if constexpr (kNumElements == 1) { + cute::SM100_TMEM_LOAD_32dp32b1x::copy(tmem_start, data[Is]...); + } else if constexpr (kNumElements == 2) { + cute::SM100_TMEM_LOAD_32dp32b2x::copy(tmem_start, data[Is]...); + } else if constexpr (kNumElements == 4) { + cute::SM100_TMEM_LOAD_32dp32b4x::copy(tmem_start, data[Is]...); + } else if constexpr (kNumElements == 8) { + cute::SM100_TMEM_LOAD_32dp32b8x::copy(tmem_start, data[Is]...); + } else if constexpr (kNumElements == 16) { + cute::SM100_TMEM_LOAD_32dp32b16x::copy(tmem_start, data[Is]...); + } else if constexpr (kNumElements == 32) { + cute::SM100_TMEM_LOAD_32dp32b32x::copy(tmem_start, data[Is]...); + } else if constexpr (kNumElements == 64) { + cute::SM100_TMEM_LOAD_32dp32b64x::copy(tmem_start, data[Is]...); + } else if constexpr (kNumElements == 128) { + cute::SM100_TMEM_LOAD_32dp32b128x::copy(tmem_start, data[Is]...); + } + }(cute::make_index_sequence{}); +#endif +} + +// Load from tensor memory with column-wise reduction, 32 data path lanes, 32-bit pattern, repeated N times. +// (https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-instructions-tcgen05-ld) +// USE_MAX: true = .max reduction, false = .min reduction +// USE_ABS: true = .abs qualifier (use absolute values for reduction) +// USE_NAN: true = .NaN qualifier (propagate NaN in reduction) +template +__device__ __forceinline__ +void tmem_ld_red_32dp32bNx(uint32_t tmem_start, void* data_, float& redval) { + uint32_t* data = (uint32_t*)data_; + static_assert(kNumElements == 2 || kNumElements == 4 || kNumElements == 8 || + kNumElements == 16 || kNumElements == 32 || kNumElements == 64 || + kNumElements == 128, + "Invalid kNumElements for tcgen05.ld.red (must be power of 2, at least 2)"); +#ifndef __VSCODE_IDE__ + static constexpr char s_max[] = ".max"; + static constexpr char s_min[] = ".min"; + static constexpr char s_abs[] = ".abs"; + static constexpr char s_nan[] = ".NaN"; + static constexpr char s_empty[] = ""; + // Operand numbering for each branch: + // outputs: %0..%{N-1} = data[0..N-1] ("=r"), %N = redval ("=f") + // inputs: %{N+1} = tmem_start ("r"), %{N+2} = redop ("C"), %{N+3} = abs ("C"), %{N+4} = nan ("C") + if constexpr (kNumElements == 2) { + // outputs: %0..%1 = data, %2 = redval; inputs: %3 = taddr, %4 = redop, %5 = abs, %6 = nan + asm volatile( + "tcgen05.ld.red.sync.aligned.32x32b.x2%4%5%6.f32" + " {%0, %1}, %2, [%3];\n" + : "=r"(data[0]), "=r"(data[1]), + "=f"(redval) + : "r"(tmem_start), + "C"(USE_MAX ? s_max : s_min), + "C"(USE_ABS ? s_abs : s_empty), + "C"(USE_NAN ? s_nan : s_empty) + ); + } else if constexpr (kNumElements == 4) { + // outputs: %0..%3 = data, %4 = redval; inputs: %5 = taddr, %6..%8 = C + asm volatile( + "tcgen05.ld.red.sync.aligned.32x32b.x4%6%7%8.f32" + " {%0, %1, %2, %3}, %4, [%5];\n" + : "=r"(data[0]), "=r"(data[1]), "=r"(data[2]), "=r"(data[3]), + "=f"(redval) + : "r"(tmem_start), + "C"(USE_MAX ? s_max : s_min), + "C"(USE_ABS ? s_abs : s_empty), + "C"(USE_NAN ? s_nan : s_empty) + ); + } else if constexpr (kNumElements == 8) { + // outputs: %0..%7 = data, %8 = redval; inputs: %9 = taddr, %10..%12 = C + asm volatile( + "tcgen05.ld.red.sync.aligned.32x32b.x8%10%11%12.f32" + " {%0, %1, %2, %3," + " %4, %5, %6, %7}, %8, [%9];\n" + : "=r"(data[0]), "=r"(data[1]), "=r"(data[2]), "=r"(data[3]), + "=r"(data[4]), "=r"(data[5]), "=r"(data[6]), "=r"(data[7]), + "=f"(redval) + : "r"(tmem_start), + "C"(USE_MAX ? s_max : s_min), + "C"(USE_ABS ? s_abs : s_empty), + "C"(USE_NAN ? s_nan : s_empty) + ); + } else if constexpr (kNumElements == 16) { + // outputs: %0..%15 = data, %16 = redval; inputs: %17 = taddr, %18..%20 = C + asm volatile( + "tcgen05.ld.red.sync.aligned.32x32b.x16%18%19%20.f32" + " {%0, %1, %2, %3," + " %4, %5, %6, %7," + " %8, %9, %10, %11," + " %12, %13, %14, %15}, %16, [%17];\n" + : "=r"(data[0]), "=r"(data[1]), "=r"(data[2]), "=r"(data[3]), + "=r"(data[4]), "=r"(data[5]), "=r"(data[6]), "=r"(data[7]), + "=r"(data[8]), "=r"(data[9]), "=r"(data[10]), "=r"(data[11]), + "=r"(data[12]), "=r"(data[13]), "=r"(data[14]), "=r"(data[15]), + "=f"(redval) + : "r"(tmem_start), + "C"(USE_MAX ? s_max : s_min), + "C"(USE_ABS ? s_abs : s_empty), + "C"(USE_NAN ? s_nan : s_empty) + ); + } else if constexpr (kNumElements == 32) { + // outputs: %0..%31 = data, %32 = redval; inputs: %33 = taddr, %34..%36 = C + asm volatile( + "tcgen05.ld.red.sync.aligned.32x32b.x32%34%35%36.f32" + " {%0, %1, %2, %3," + " %4, %5, %6, %7," + " %8, %9, %10, %11," + " %12, %13, %14, %15," + " %16, %17, %18, %19," + " %20, %21, %22, %23," + " %24, %25, %26, %27," + " %28, %29, %30, %31}, %32, [%33];\n" + : "=r"(data[0]), "=r"(data[1]), "=r"(data[2]), "=r"(data[3]), + "=r"(data[4]), "=r"(data[5]), "=r"(data[6]), "=r"(data[7]), + "=r"(data[8]), "=r"(data[9]), "=r"(data[10]), "=r"(data[11]), + "=r"(data[12]), "=r"(data[13]), "=r"(data[14]), "=r"(data[15]), + "=r"(data[16]), "=r"(data[17]), "=r"(data[18]), "=r"(data[19]), + "=r"(data[20]), "=r"(data[21]), "=r"(data[22]), "=r"(data[23]), + "=r"(data[24]), "=r"(data[25]), "=r"(data[26]), "=r"(data[27]), + "=r"(data[28]), "=r"(data[29]), "=r"(data[30]), "=r"(data[31]), + "=f"(redval) + : "r"(tmem_start), + "C"(USE_MAX ? s_max : s_min), + "C"(USE_ABS ? s_abs : s_empty), + "C"(USE_NAN ? s_nan : s_empty) + ); + } else if constexpr (kNumElements == 64) { + // outputs: %0..%63 = data, %64 = redval; inputs: %65 = taddr, %66..%68 = C + asm volatile( + "tcgen05.ld.red.sync.aligned.32x32b.x64%66%67%68.f32" + " {%0, %1, %2, %3," + " %4, %5, %6, %7," + " %8, %9, %10, %11," + " %12, %13, %14, %15," + " %16, %17, %18, %19," + " %20, %21, %22, %23," + " %24, %25, %26, %27," + " %28, %29, %30, %31," + " %32, %33, %34, %35," + " %36, %37, %38, %39," + " %40, %41, %42, %43," + " %44, %45, %46, %47," + " %48, %49, %50, %51," + " %52, %53, %54, %55," + " %56, %57, %58, %59," + " %60, %61, %62, %63}, %64, [%65];\n" + : "=r"(data[0]), "=r"(data[1]), "=r"(data[2]), "=r"(data[3]), + "=r"(data[4]), "=r"(data[5]), "=r"(data[6]), "=r"(data[7]), + "=r"(data[8]), "=r"(data[9]), "=r"(data[10]), "=r"(data[11]), + "=r"(data[12]), "=r"(data[13]), "=r"(data[14]), "=r"(data[15]), + "=r"(data[16]), "=r"(data[17]), "=r"(data[18]), "=r"(data[19]), + "=r"(data[20]), "=r"(data[21]), "=r"(data[22]), "=r"(data[23]), + "=r"(data[24]), "=r"(data[25]), "=r"(data[26]), "=r"(data[27]), + "=r"(data[28]), "=r"(data[29]), "=r"(data[30]), "=r"(data[31]), + "=r"(data[32]), "=r"(data[33]), "=r"(data[34]), "=r"(data[35]), + "=r"(data[36]), "=r"(data[37]), "=r"(data[38]), "=r"(data[39]), + "=r"(data[40]), "=r"(data[41]), "=r"(data[42]), "=r"(data[43]), + "=r"(data[44]), "=r"(data[45]), "=r"(data[46]), "=r"(data[47]), + "=r"(data[48]), "=r"(data[49]), "=r"(data[50]), "=r"(data[51]), + "=r"(data[52]), "=r"(data[53]), "=r"(data[54]), "=r"(data[55]), + "=r"(data[56]), "=r"(data[57]), "=r"(data[58]), "=r"(data[59]), + "=r"(data[60]), "=r"(data[61]), "=r"(data[62]), "=r"(data[63]), + "=f"(redval) + : "r"(tmem_start), + "C"(USE_MAX ? s_max : s_min), + "C"(USE_ABS ? s_abs : s_empty), + "C"(USE_NAN ? s_nan : s_empty) + ); + } else if constexpr (kNumElements == 128) { + // outputs: %0..%127 = data, %128 = redval; inputs: %129 = taddr, %130..%132 = C + asm volatile( + "tcgen05.ld.red.sync.aligned.32x32b.x128%130%131%132.f32" + " {%0, %1, %2, %3," + " %4, %5, %6, %7," + " %8, %9, %10, %11," + " %12, %13, %14, %15," + " %16, %17, %18, %19," + " %20, %21, %22, %23," + " %24, %25, %26, %27," + " %28, %29, %30, %31," + " %32, %33, %34, %35," + " %36, %37, %38, %39," + " %40, %41, %42, %43," + " %44, %45, %46, %47," + " %48, %49, %50, %51," + " %52, %53, %54, %55," + " %56, %57, %58, %59," + " %60, %61, %62, %63," + " %64, %65, %66, %67," + " %68, %69, %70, %71," + " %72, %73, %74, %75," + " %76, %77, %78, %79," + " %80, %81, %82, %83," + " %84, %85, %86, %87," + " %88, %89, %90, %91," + " %92, %93, %94, %95," + " %96, %97, %98, %99," + " %100, %101, %102, %103," + " %104, %105, %106, %107," + " %108, %109, %110, %111," + " %112, %113, %114, %115," + " %116, %117, %118, %119," + " %120, %121, %122, %123," + " %124, %125, %126, %127}, %128, [%129];\n" + : "=r"(data[0]), "=r"(data[1]), "=r"(data[2]), "=r"(data[3]), + "=r"(data[4]), "=r"(data[5]), "=r"(data[6]), "=r"(data[7]), + "=r"(data[8]), "=r"(data[9]), "=r"(data[10]), "=r"(data[11]), + "=r"(data[12]), "=r"(data[13]), "=r"(data[14]), "=r"(data[15]), + "=r"(data[16]), "=r"(data[17]), "=r"(data[18]), "=r"(data[19]), + "=r"(data[20]), "=r"(data[21]), "=r"(data[22]), "=r"(data[23]), + "=r"(data[24]), "=r"(data[25]), "=r"(data[26]), "=r"(data[27]), + "=r"(data[28]), "=r"(data[29]), "=r"(data[30]), "=r"(data[31]), + "=r"(data[32]), "=r"(data[33]), "=r"(data[34]), "=r"(data[35]), + "=r"(data[36]), "=r"(data[37]), "=r"(data[38]), "=r"(data[39]), + "=r"(data[40]), "=r"(data[41]), "=r"(data[42]), "=r"(data[43]), + "=r"(data[44]), "=r"(data[45]), "=r"(data[46]), "=r"(data[47]), + "=r"(data[48]), "=r"(data[49]), "=r"(data[50]), "=r"(data[51]), + "=r"(data[52]), "=r"(data[53]), "=r"(data[54]), "=r"(data[55]), + "=r"(data[56]), "=r"(data[57]), "=r"(data[58]), "=r"(data[59]), + "=r"(data[60]), "=r"(data[61]), "=r"(data[62]), "=r"(data[63]), + "=r"(data[64]), "=r"(data[65]), "=r"(data[66]), "=r"(data[67]), + "=r"(data[68]), "=r"(data[69]), "=r"(data[70]), "=r"(data[71]), + "=r"(data[72]), "=r"(data[73]), "=r"(data[74]), "=r"(data[75]), + "=r"(data[76]), "=r"(data[77]), "=r"(data[78]), "=r"(data[79]), + "=r"(data[80]), "=r"(data[81]), "=r"(data[82]), "=r"(data[83]), + "=r"(data[84]), "=r"(data[85]), "=r"(data[86]), "=r"(data[87]), + "=r"(data[88]), "=r"(data[89]), "=r"(data[90]), "=r"(data[91]), + "=r"(data[92]), "=r"(data[93]), "=r"(data[94]), "=r"(data[95]), + "=r"(data[96]), "=r"(data[97]), "=r"(data[98]), "=r"(data[99]), + "=r"(data[100]), "=r"(data[101]), "=r"(data[102]), "=r"(data[103]), + "=r"(data[104]), "=r"(data[105]), "=r"(data[106]), "=r"(data[107]), + "=r"(data[108]), "=r"(data[109]), "=r"(data[110]), "=r"(data[111]), + "=r"(data[112]), "=r"(data[113]), "=r"(data[114]), "=r"(data[115]), + "=r"(data[116]), "=r"(data[117]), "=r"(data[118]), "=r"(data[119]), + "=r"(data[120]), "=r"(data[121]), "=r"(data[122]), "=r"(data[123]), + "=r"(data[124]), "=r"(data[125]), "=r"(data[126]), "=r"(data[127]), + "=f"(redval) + : "r"(tmem_start), + "C"(USE_MAX ? s_max : s_min), + "C"(USE_ABS ? s_abs : s_empty), + "C"(USE_NAN ? s_nan : s_empty) + ); + } +#endif +} + +// Load from tensor memory, 16 data path lanes, 128-bit pattern, repeated N times. (https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-instructions-tcgen05-ld) +template +__device__ __forceinline__ +void tmem_ld_16dp128bNx(uint32_t tmem_start, void* data_) { + uint32_t* data = (uint32_t*)data_; + static_assert(kNumReplications == 1 || kNumReplications == 2 || kNumReplications == 4 || kNumReplications == 8 || kNumReplications == 16 || kNumReplications == 32 || kNumReplications == 64, "Invalid kNumReplications"); +#ifndef __VSCODE_IDE__ + [&](cute::index_sequence) { + if constexpr (kNumReplications == 1) { + cute::SM100_TMEM_LOAD_16dp128b1x::copy(tmem_start, data[Is]...); + } else if constexpr (kNumReplications == 2) { + cute::SM100_TMEM_LOAD_16dp128b2x::copy(tmem_start, data[Is]...); + } else if constexpr (kNumReplications == 4) { + cute::SM100_TMEM_LOAD_16dp128b4x::copy(tmem_start, data[Is]...); + } else if constexpr (kNumReplications == 8) { + cute::SM100_TMEM_LOAD_16dp128b8x::copy(tmem_start, data[Is]...); + } else if constexpr (kNumReplications == 16) { + cute::SM100_TMEM_LOAD_16dp128b16x::copy(tmem_start, data[Is]...); + } else if constexpr (kNumReplications == 32) { + cute::SM100_TMEM_LOAD_16dp128b32x::copy(tmem_start, data[Is]...); + } else if constexpr (kNumReplications == 64) { + cute::SM100_TMEM_LOAD_16dp128b64x::copy(tmem_start, data[Is]...); + } + }(cute::make_index_sequence{}); +#endif +} + +// Load from tensor memory, 16 data path lanes, 256-bit pattern, repeated N times. (https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-instructions-tcgen05-ld) +template +__device__ __forceinline__ +void tmem_ld_16dp256bNx(uint32_t tmem_start, void* data_) { + uint32_t* data = (uint32_t*)data_; + static_assert(kNumReplications == 1 || kNumReplications == 2 || kNumReplications == 4 || kNumReplications == 8 || kNumReplications == 16 || kNumReplications == 32, "Invalid kNumReplications"); +#ifndef __VSCODE_IDE__ + [&](cute::index_sequence) { + if constexpr (kNumReplications == 1) { + cute::SM100_TMEM_LOAD_16dp256b1x::copy(tmem_start, data[Is]...); + } else if constexpr (kNumReplications == 2) { + cute::SM100_TMEM_LOAD_16dp256b2x::copy(tmem_start, data[Is]...); + } else if constexpr (kNumReplications == 4) { + cute::SM100_TMEM_LOAD_16dp256b4x::copy(tmem_start, data[Is]...); + } else if constexpr (kNumReplications == 8) { + cute::SM100_TMEM_LOAD_16dp256b8x::copy(tmem_start, data[Is]...); + } else if constexpr (kNumReplications == 16) { + cute::SM100_TMEM_LOAD_16dp256b16x::copy(tmem_start, data[Is]...); + } else if constexpr (kNumReplications == 32) { + cute::SM100_TMEM_LOAD_16dp256b32x::copy(tmem_start, data[Is]...); + } + }(cute::make_index_sequence{}); +#endif +} + +// Store into tensor memory, 32 data path lanes, 32-bit pattern, repeated N times. (https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-instructions-tcgen05-st) +template +__device__ __forceinline__ +void tmem_st_32dp32bNx(uint32_t tmem_start, void const* data_) { + uint32_t const* data = (uint32_t const*)data_; + static_assert(kNumElements == 1 || kNumElements == 2 || kNumElements == 4 || kNumElements == 8 || kNumElements == 16 || kNumElements == 32 || kNumElements == 64 || kNumElements == 128, "Invalid kNumElements"); +#ifndef __VSCODE_IDE__ + [&](cute::index_sequence) { + if constexpr (kNumElements == 1) { + cute::SM100_TMEM_STORE_32dp32b1x::copy(data[Is]..., tmem_start); + } else if constexpr (kNumElements == 2) { + cute::SM100_TMEM_STORE_32dp32b2x::copy(data[Is]..., tmem_start); + } else if constexpr (kNumElements == 4) { + cute::SM100_TMEM_STORE_32dp32b4x::copy(data[Is]..., tmem_start); + } else if constexpr (kNumElements == 8) { + cute::SM100_TMEM_STORE_32dp32b8x::copy(data[Is]..., tmem_start); + } else if constexpr (kNumElements == 16) { + cute::SM100_TMEM_STORE_32dp32b16x::copy(data[Is]..., tmem_start); + } else if constexpr (kNumElements == 32) { + cute::SM100_TMEM_STORE_32dp32b32x::copy(data[Is]..., tmem_start); + } else if constexpr (kNumElements == 64) { + cute::SM100_TMEM_STORE_32dp32b64x::copy(data[Is]..., tmem_start); + } else if constexpr (kNumElements == 128) { + cute::SM100_TMEM_STORE_32dp32b128x::copy(data[Is]..., tmem_start); + } + }(cute::make_index_sequence{}); +#endif +} + +// Decompose a non-power-of-2 replication count into a sequence of power-of-2 calls. +// +// This helper splits kNumReplications into its constituent powers of 2 (from highest to lowest) +// and issues one call to tmem_ld_st_fn.template operator()() for each, automatically +// advancing the tmem address by kNumElemsPerUnit columns and the data pointer by kNumElemsPerUnit +// uint32_t elements per unit. +// +// Template parameters: +// kNumReplications — total number of replications (may be non-power-of-2) +// kNumElemsPerUnit — number of tmem columns (and uint32_t data elements) consumed per unit +// (e.g. 1 for 32dp32b, 2 for 16dp128b, 4 for 16dp256b) +// F — a callable with a template operator(): f.template operator()(uint32_t, void*) +// +// Usage example (with a template lambda): +// tmem_ld_st_decomposed<6, 1>(tmem_start, data_ptr, +// [](uint32_t ts, void* d) { tmem_ld_32dp32bNx(ts, d); }); +// // Equivalent to: +// // tmem_ld_32dp32bNx<4>(tmem_start, data_ptr); +// // tmem_ld_32dp32bNx<2>(tmem_start + 4, (void*)((uint32_t*)data_ptr + 4)); +template +__device__ __forceinline__ +void tmem_ld_st_decomposed(uint32_t tmem_start, void* data, F&& tmem_ld_st_fn) { + if constexpr (kNumReplications == 0) { + // Base case: nothing to do + return; + } else { + // Largest power-of-2 <= kNumReplications + constexpr int kHighBit = 1 << (31 - __builtin_clz(kNumReplications)); + tmem_ld_st_fn.template operator()(tmem_start, data); + // Recurse on the remainder + constexpr int kRemaining = kNumReplications - kHighBit; + if constexpr (kRemaining > 0) { + tmem_ld_st_decomposed( + tmem_start + kHighBit * kNumElemsPerUnit, + (void*)((uint32_t*)data + kHighBit * kNumElemsPerUnit), + tmem_ld_st_fn + ); + } + } +} + +} diff --git a/python/sglang/kernels/jit/csrc/deepselect/vendor/3rdparty/kerutils/include/kerutils/device/cuda/sm80/helpers.cuh b/python/sglang/kernels/jit/csrc/deepselect/vendor/3rdparty/kerutils/include/kerutils/device/cuda/sm80/helpers.cuh new file mode 100644 index 0000000000..c3cad8372a --- /dev/null +++ b/python/sglang/kernels/jit/csrc/deepselect/vendor/3rdparty/kerutils/include/kerutils/device/cuda/sm80/helpers.cuh @@ -0,0 +1,55 @@ +#pragma once + +#include "kerutils/device/cuda/common.h" +#include "kerutils/device/cuda/sm80/intrinsics.cuh" + +namespace kerutils { + +// Retrieve the value of `%smid` and check its range +CUTE_DEVICE +uint32_t get_sm_id_with_range_check(uint32_t num_physical_sms) { + uint32_t sm_id = get_sm_id(); + if (!(sm_id < num_physical_sms)) { + trap(); + } + return sm_id; +} + +#ifndef KU_TRAP_ONLY_DEVICE_ASSERT +#define KU_TRAP_ONLY_DEVICE_ASSERT(cond) \ +do { \ + if (not (cond)) \ + asm("trap;"); \ +} while (0) +#endif + +// Construct a `float2` from a single `float` by duplicating the value +CUTE_DEVICE +float2 float2float2(const float &x) { + return float2 {x, x}; +} + +CUTE_DEVICE +void st_shared(void* ptr, __int128_t val) { + asm volatile("st.shared.b128 [%0], %1;" :: "r"(cute::cast_smem_ptr_to_uint(ptr)), "q"(val)); +} + +CUTE_DEVICE +void st_shared(void* ptr, float4 val) { + st_shared(ptr, *(__int128_t*)&val); +} + +CUTE_DEVICE +__int128_t ld_shared(const void* ptr) { + __int128_t val; + asm volatile("ld.shared.b128 %0, [%1];" : "=q"(val) : "r"(cute::cast_smem_ptr_to_uint(ptr))); + return val; +} + +CUTE_DEVICE +float4 ld_shared_float4(const void* ptr) { + __int128_t temp = ld_shared(ptr); + return *(float4*)&temp; +} + +} diff --git a/python/sglang/kernels/jit/csrc/deepselect/vendor/3rdparty/kerutils/include/kerutils/device/cuda/sm80/intrinsics.cuh b/python/sglang/kernels/jit/csrc/deepselect/vendor/3rdparty/kerutils/include/kerutils/device/cuda/sm80/intrinsics.cuh new file mode 100644 index 0000000000..01937ed232 --- /dev/null +++ b/python/sglang/kernels/jit/csrc/deepselect/vendor/3rdparty/kerutils/include/kerutils/device/cuda/sm80/intrinsics.cuh @@ -0,0 +1,160 @@ +#pragma once + +#include "kerutils/device/cuda/common.h" + +namespace kerutils { + +// cp.async.cg (cache global) with prefetch and predicate (https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-cp-async) +template +CUTE_DEVICE +void cp_async_cacheglobal(const void* src, void* dst, bool pred=true) { + uint32_t dst_addr = cute::cast_smem_ptr_to_uint(dst); + if constexpr (PREFETCH_SIZE == PrefetchSize::B64) { + asm volatile("cp.async.cg.shared.global.L2::64B [%0], [%1], 16, %2;\n" + :: "r"(dst_addr), + "l"(src), + "r"(pred?16:0)); + } else if constexpr (PREFETCH_SIZE == PrefetchSize::B128) { + asm volatile("cp.async.cg.shared.global.L2::128B [%0], [%1], 16, %2;\n" + :: "r"(dst_addr), + "l"(src), + "r"(pred?16:0)); + } else if constexpr (PREFETCH_SIZE == PrefetchSize::B256) { + asm volatile("cp.async.cg.shared.global.L2::256B [%0], [%1], 16, %2;\n" + :: "r"(dst_addr), + "l"(src), + "r"(pred?16:0)); + } else { + static_assert(PREFETCH_SIZE == PrefetchSize::B64 || + PREFETCH_SIZE == PrefetchSize::B128 || + PREFETCH_SIZE == PrefetchSize::B256, + "Unsupported prefetch size for cp_async_cacheglobal."); + } +} + +// Create fraction-based cache policy (https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-createpolicy) +template +CUTE_DEVICE +int64_t create_fraction_based_cache_policy(float fraction = 1.0f) { + int64_t result; + #define EMIT(PRIMARY_PRIORITY_STR, SECONDARY_PRIORITY_STR) \ + asm volatile( \ + "createpolicy.fractional.L2::" PRIMARY_PRIORITY_STR ".L2::" SECONDARY_PRIORITY_STR ".b64 %0, %1;\n" \ + : "=l"(result) \ + : "f"(fraction) \ + ); + #define EMIT2(PRIMARY_PRIORITY_STR) \ + { \ + if constexpr (SECONDARY_PRIORITY == CacheHint::EVICT_FIRST) { \ + EMIT(PRIMARY_PRIORITY_STR, "evict_first") \ + } else if constexpr (SECONDARY_PRIORITY == CacheHint::EVICT_UNCHANGED) { \ + EMIT(PRIMARY_PRIORITY_STR, "evict_unchanged") \ + } else { \ + static_assert(SECONDARY_PRIORITY == CacheHint::EVICT_FIRST || \ + SECONDARY_PRIORITY == CacheHint::EVICT_UNCHANGED, \ + "Unsupported secondary cache hint for create_fraction_based_cache_policy."); \ + } \ + } + if constexpr (PRIMARY_PRIORITY == CacheHint::EVICT_FIRST) { + EMIT2("evict_first"); + } else if constexpr (PRIMARY_PRIORITY == CacheHint::EVICT_NORMAL) { + EMIT2("evict_normal"); + } else if constexpr (PRIMARY_PRIORITY == CacheHint::EVICT_LAST) { + EMIT2("evict_last"); + } else if constexpr (PRIMARY_PRIORITY == CacheHint::EVICT_UNCHANGED) { + EMIT2("evict_unchanged"); + } else { + static_assert(PRIMARY_PRIORITY == CacheHint::EVICT_FIRST || + PRIMARY_PRIORITY == CacheHint::EVICT_NORMAL || + PRIMARY_PRIORITY == CacheHint::EVICT_LAST || + PRIMARY_PRIORITY == CacheHint::EVICT_UNCHANGED, + "Unsupported primary cache hint for create_fraction_based_cache_policy."); + } + #undef EMIT + #undef EMIT2 + return result; +} + +// Create a simple cache policy (equivalent to create_fraction_based_cache_policy(1.0f)) +// The same as cute::TMA::CacheHintSmXX +template +CUTE_DEVICE +constexpr int64_t create_simple_cache_policy() { + if constexpr (CACHE_HINT == CacheHint::EVICT_FIRST) { + return 0x12F0000000000000; // Result of createpolicy.fractional.L2::evict_first.b64 + } else if constexpr (CACHE_HINT == CacheHint::EVICT_NORMAL) { + return 0x1000000000000000; // Copied from CuTe. Unsure about the exact meaning. (TODO Change to 0x16F0000000000000?) + } else if constexpr (CACHE_HINT == CacheHint::EVICT_LAST) { + return 0x14F0000000000000; // Result of createpolicy.fractional.L2::evict_last.b64 + } else { + static_assert(CACHE_HINT == CacheHint::EVICT_FIRST || + CACHE_HINT == CacheHint::EVICT_NORMAL || + CACHE_HINT == CacheHint::EVICT_LAST, + "Unsupported cache hint for create_simple_cache_policy."); + } +} + +// AtomicAdd (https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-red) +CUTE_DEVICE +void atomicadd_f32_with_policy_and_pred(void* global_addr, const float &data, int64_t cache_policy, uint32_t pred = true) { + asm volatile( + "{\n\t" + ".reg .pred p;\n\t" + "setp.eq.u32 p, %3, 1;\n\t" + "@p red.relaxed.gpu.global.add.L2::cache_hint.f32 [%1], %0, %2; \n\t" + "}" + : + : "f"(data), + "l"((int64_t)global_addr), "l"(cache_policy), "r"(pred) + ); +} + +// Get the id of the current SM +// About %smid (https://docs.nvidia.com/cuda/parallel-thread-execution/#special-registers-smid): PTX document says that %smid ranges from 0 to %nsmid-1, while "The SM identifier numbering is not guaranteed to be contiguous, so %nsmid may be larger than the physical number of SMs in the device.". However, result shows that, at least for sm90 and sm100, %nsmid is the number of physical SMs - 1. For the sake of safety, I recommend you to check the return of get_sm_id manually or call `get_sm_id_with_range_check()` defined in `device/sm80/helpers.cuh`. +// Besides, PTX document also says that this number may change due to preemption, but currently this never happens according to [DATEN GELÖSCHT] +CUTE_DEVICE +uint32_t get_sm_id() { + uint32_t ret; + asm volatile("mov.u32 %0, %%smid;\n" : "=r"(ret)); + return ret; +} + +// trap (https://docs.nvidia.com/cuda/parallel-thread-execution/#miscellaneous-instructions-trap) +CUTE_DEVICE +void trap() { + asm volatile("trap;\n"); +} + +// LDG.128 or LDG.128 with non-coherent cache (https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-ld) +// We use macro instead of function here, since we need a multi-level recursive dispatch based on template parameters if using function +// NC_STR should be either "" or ".nc" +// L1_CACHE_HINT_STR should be either "evict_first", "evict_normal", "evict_last", "evict_unchanged", or "no_allocate" +// L2_PREFETCH_SIZE_STR should be either "64B", "128B", or "256B" +// L2 cache hint is not supported since it's only supported for LDG.256 +#define KU_LDG_128(global_addr, result, NC_STR, L1_CACHE_HINT_STR, L2_PREFETCH_SIZE_STR) \ + { \ + static_assert(std::is_pointer_v || std::is_array_v, "`global_addr` must be a pointer"); \ + static_assert(std::is_pointer_v || std::is_array_v, "`result` must be a pointer"); \ + uint64_t* result_as_uint64_ptr = (uint64_t*)(result); \ + asm volatile( \ + "ld.global" NC_STR ".L1::" L1_CACHE_HINT_STR ".L2::" L2_PREFETCH_SIZE_STR ".v2.u64 {%0, %1}, [%2];\n" \ + : "=l"(result_as_uint64_ptr[0]), "=l"(result_as_uint64_ptr[1]) \ + : "l"(global_addr) \ + ); \ + } + +// STG.128 (https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-st) +// L2_CACHE_HINT_STR should be ".L2::evict_XXX" (only available on sm100+) or "" +#define KU_STG_128(global_addr, src, L1_CACHE_HINT_STR, L2_CACHE_HINT_STR) \ + { \ + static_assert(std::is_pointer_v || std::is_array_v, "`global_addr` must be a pointer"); \ + static_assert(std::is_pointer_v || std::is_array_v, "`src` must be a pointer"); \ + uint64_t const* src_as_uint64_ptr = (uint64_t const*)(src); \ + asm volatile( \ + "st.global.L1::" L1_CACHE_HINT_STR L2_CACHE_HINT_STR ".v2.u64 [%0], {%1, %2};\n" \ + : \ + : "l"(global_addr), "l"(src_as_uint64_ptr[0]), "l"(src_as_uint64_ptr[1]) \ + ); \ + } + +} diff --git a/python/sglang/kernels/jit/csrc/deepselect/vendor/3rdparty/kerutils/include/kerutils/device/cuda/sm90/intrinsics.cuh b/python/sglang/kernels/jit/csrc/deepselect/vendor/3rdparty/kerutils/include/kerutils/device/cuda/sm90/intrinsics.cuh new file mode 100644 index 0000000000..f37b04b382 --- /dev/null +++ b/python/sglang/kernels/jit/csrc/deepselect/vendor/3rdparty/kerutils/include/kerutils/device/cuda/sm90/intrinsics.cuh @@ -0,0 +1,107 @@ +#pragma once + +#include "kerutils/device/cuda/common.h" + +namespace kerutils { + +// st.async (https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-st-async) +template +CUTE_DEVICE +static void st_async(void* dst_ptr, const T& data, transac_bar_t &mbar) { + static_assert(sizeof(T) == 16, "Data type must be 16 bytes (128 bits) for st_async."); + long2 data_long2 = *reinterpret_cast(&data); + uint32_t dst_addr = cute::cast_smem_ptr_to_uint(dst_ptr); + uint32_t mbar_addr = cute::cast_smem_ptr_to_uint(&mbar); + asm volatile ( + "st.async.weak.shared::cluster.mbarrier::complete_tx::bytes.v2.s64 [%0], {%1, %2}, [%3]; \n" + : + : "r"(dst_addr), "l"(data_long2.x), "l"(data_long2.y), "r"(mbar_addr) + ); +} + +static constexpr int PEER_ADDR_MASK = 16777216; + +// Given an address in the current CTA, return the corresponding address in the peer CTA +template +CUTE_DEVICE +T* get_peer_addr(const T* p) { + return (T*)((int64_t)(p) ^ PEER_ADDR_MASK); +} + +// Given an address in the current CTA, return the corresponding address in the peer CTA (if the current CTA_id%2 == 1) or the address itself (if CTA_id%2 == 0) +template +CUTE_DEVICE +T* get_cta0_addr(const T* p) { + constexpr int CTA0_ADDR_MASK = 0xFEFFFFFF; + return (T*)((int64_t)(p) & CTA0_ADDR_MASK); +} + +// TMA bulk reduce add (cp.reduce.async.bulk), shared to global, float32, add. (https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-cp-reduce-async-bulk) +CUTE_DEVICE +void tma_bulk_reduce_add(void const* src_ptr, void* dst_ptr, int32_t store_bytes) { + uint32_t smem_int_ptr = cute::cast_smem_ptr_to_uint(src_ptr); + asm volatile("cp.reduce.async.bulk.global.shared::cta.bulk_group.add.f32 [%0], [%1], %2;\n" + : + : "l"(dst_ptr), "r"(smem_int_ptr), "r"(store_bytes) + : "memory"); +} + +// Cluster barrier arrive with .release modifier. (https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-barrier-cluster) +CUTE_DEVICE +void barrier_cluster_arrive_release() { + asm volatile("barrier.cluster.arrive.release;" : : : "memory"); +} + +// Cluster barrier arrive with .relaxed modifier. (https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-barrier-cluster) +CUTE_DEVICE +void barrier_cluster_arrive_relaxed() { + asm volatile("barrier.cluster.arrive.relaxed;" : : :); +} + +// Cluster barrier wait with .acquire modifier. (https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-barrier-cluster) +CUTE_DEVICE +void barrier_cluster_wait_acquire() { + asm volatile("barrier.cluster.wait.acquire;" : : : "memory"); +} + +// mbarrier.arrive with .relaxed.cluster qualifier (https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-mbarrier-arrive) +CUTE_DEVICE +void mbarrier_arrive_relaxed_cluster(transac_bar_t &mbar) { + uint32_t smem_addr = cute::cast_smem_ptr_to_uint(&mbar); + asm volatile( + "{\n\t" + "mbarrier.arrive.relaxed.cluster.shared::cta.b64 _, [%0];\n\t" + "}" + : + : "r"(smem_addr)); +} + +// AtomicAdd with v4.f32 type (https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-red) +CUTE_DEVICE +void atomicadd_f32x4_with_policy_and_pred(void* global_addr, const float4 &data, int64_t cache_policy, uint32_t pred = true) { + asm volatile( + "{\n\t" + ".reg .pred p;\n\t" + "setp.eq.u32 p, %6, 1;\n\t" + "@p red.relaxed.gpu.global.add.L2::cache_hint.v4.f32 [%4], {%0, %1, %2, %3}, %5; \n\t" + "}" + : + : "f"(data.x), "f"(data.y), "f"(data.z), "f"(data.w), + "l"((int64_t)global_addr), "l"(cache_policy), "r"(pred) + ); +} + +// cp.async.bulk, from .shared::cta to .shared::cluster (https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-cp-async-bulk) +CUTE_DEVICE +void cp_async_bulk_shared_cta_to_shared_cluster(void* dst_ptr, const void* src_ptr, int32_t load_bytes, transac_bar_t &mbar) { + uint32_t dst_smem_addr = cute::cast_smem_ptr_to_uint(dst_ptr); + uint32_t src_smem_addr = cute::cast_smem_ptr_to_uint(src_ptr); + uint32_t mbar_addr = cute::cast_smem_ptr_to_uint(&mbar); + asm volatile( + "cp.async.bulk.shared::cluster.shared::cta.mbarrier::complete_tx::bytes [%0], [%1], %2, [%3]; \n" + : + : "r"(dst_smem_addr), "r"(src_smem_addr), "r"(load_bytes), "r"(mbar_addr) + ); +} + +} diff --git a/python/sglang/kernels/jit/csrc/deepselect/vendor/3rdparty/kerutils/include/kerutils/device/device.cuh b/python/sglang/kernels/jit/csrc/deepselect/vendor/3rdparty/kerutils/include/kerutils/device/device.cuh new file mode 100644 index 0000000000..edc5af5c12 --- /dev/null +++ b/python/sglang/kernels/jit/csrc/deepselect/vendor/3rdparty/kerutils/include/kerutils/device/device.cuh @@ -0,0 +1,11 @@ +#pragma once + +#include "kerutils/common/common.h" + +#ifdef KERUTILS_IS_BUILD_ON_CUDA +#include "cuda/common.h" +#include "cuda/sm80/intrinsics.cuh" +#include "cuda/sm80/helpers.cuh" +#include "cuda/sm90/intrinsics.cuh" +#include "cuda/sm100/intrinsics.cuh" +#endif diff --git a/python/sglang/kernels/jit/csrc/deepselect/vendor/3rdparty/kerutils/include/kerutils/host/host.h b/python/sglang/kernels/jit/csrc/deepselect/vendor/3rdparty/kerutils/include/kerutils/host/host.h new file mode 100644 index 0000000000..46617b3034 --- /dev/null +++ b/python/sglang/kernels/jit/csrc/deepselect/vendor/3rdparty/kerutils/include/kerutils/host/host.h @@ -0,0 +1,282 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#ifdef KERUTILS_IS_BUILD_ON_CUDA +#include +#include +#endif + +#include "kerutils/common/common.h" + +namespace kerutils { + +class KUException final : public std::exception { + std::string message = {}; + +public: + template + explicit KUException(const char *name, const char* file, const int line, Args&&... args) { + std::ostringstream oss; + + oss << name << " error (" << file << ":" << line << "): "; + (oss << ... << args); + message = oss.str(); + } + + const char *what() const noexcept override { + return message.c_str(); + } +}; + +#define THROW_KU_EXCEPTION(name, ...) \ + throw kerutils::KUException(name, __FILE__, __LINE__, __VA_ARGS__) + +// This `KU_ASSERT` is triggered no matter if the code is compiled with `-DNDEBUG` or not. +#define KU_ASSERT(cond, ...) \ + do { \ + if (not (cond)) { \ + char _ku_buf[1024]; \ + int _ku_len = snprintf(_ku_buf, sizeof(_ku_buf), \ + "Assertion `%s` failed (%s:%d)", #cond, __FILE__, __LINE__); \ + __VA_OPT__( \ + _ku_len += snprintf(_ku_buf + _ku_len, sizeof(_ku_buf) - _ku_len, \ + ": " __VA_ARGS__); \ + ) \ + fprintf(stderr, "%s\n", _ku_buf); \ + THROW_KU_EXCEPTION("Assertion", _ku_buf); \ + } \ + } while(0) + +#ifdef KERUTILS_IS_BUILD_ON_CUDA + +#define KU_CUDA_CHECK(call) \ +do { \ + cudaError_t status_ = (call); \ + if (status_ != cudaSuccess) { \ + char _ku_buf[1024]; \ + snprintf(_ku_buf, sizeof(_ku_buf), "CUDA error (%s:%d): %s", __FILE__, __LINE__, cudaGetErrorString(status_)); \ + fprintf(stderr, "%s\n", _ku_buf); \ + THROW_KU_EXCEPTION("CUDA", _ku_buf); \ + } \ +} while(0) + +#define KU_CUTLASS_CHECK(call) \ +do { \ + cutlass::Status status_ = (call); \ + if (status_ != cutlass::Status::kSuccess) { \ + char _ku_buf[1024]; \ + snprintf(_ku_buf, sizeof(_ku_buf), "CUTLASS error (%s:%d): %d", __FILE__, __LINE__, static_cast(status_)); \ + fprintf(stderr, "%s\n", _ku_buf); \ + THROW_KU_EXCEPTION("CUTLASS", _ku_buf); \ + } \ +} while(0) + +#define KU_CHECK_KERNEL_LAUNCH() KU_CUDA_CHECK(cudaGetLastError()) + +template +inline __host__ __device__ constexpr T ceil_div(const T &a, const T &b) { + return (a + b - 1) / b; +} + +template +inline __host__ __device__ constexpr T ceil(const T &a, const T &b) { + return (a + b - 1) / b * b; +} + +template +inline __host__ __device__ constexpr T find_next_power_of_2(const T& x) { + if (x <= LOWER_BOUND) + return LOWER_BOUND; + return find_next_power_of_2(x); +} + +// A wrapper for make_tensor_map +static inline CUtensorMap make_tensor_map( + const std::vector &size, + const std::vector &strides, // PAY ATTENTION: In BYTES + const std::vector &box_size, + void* global_ptr, + CUtensorMapDataType data_type, + CUtensorMapSwizzle swizzle_mode, + CUtensorMapL2promotion l2_promotion, + CUtensorMapInterleave interleave_mode = CUtensorMapInterleave::CU_TENSOR_MAP_INTERLEAVE_NONE, + CUtensorMapFloatOOBfill oob_fill = CUtensorMapFloatOOBfill::CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE, + const std::vector &element_strides_ = {} +) { + int dim = size.size(); + KU_ASSERT(dim >= 1); + + std::vector element_strides; + if (element_strides_.empty()) { + for (int i = 0; i < dim; ++i) + element_strides.push_back(1); + } else { + element_strides = element_strides_; + } + KU_ASSERT(strides.size() == (uint32_t)dim-1 && box_size.size() == (uint32_t)dim && element_strides.size() == (uint32_t)dim); + + auto call_cuTensorMapEncodeTiled = [&](Args... args) { + cudaDriverEntryPointQueryResult cuda_status; + void* pfn = nullptr; +#if (__CUDACC_VER_MAJOR__ > 12) + KU_CUDA_CHECK(cudaGetDriverEntryPointByVersion( + "cuTensorMapEncodeTiled", + &pfn, 12000, + cudaEnableDefault, + &cuda_status)); +#else + KU_CUDA_CHECK(cudaGetDriverEntryPoint( + "cuTensorMapEncodeTiled", + &pfn, + cudaEnableDefault, + &cuda_status)); +#endif + if (cuda_status != cudaDriverEntryPointSuccess) { + KU_ASSERT(false, "Failed to load `cuTensorMapEncodeTiled`. cuda_status = %d", cuda_status); + } + return reinterpret_cast(pfn)(args...); \ + }; + + CUtensorMap result; + CUresult ret_code = call_cuTensorMapEncodeTiled( + &result, + data_type, + dim, + global_ptr, + size.data(), + strides.data(), + box_size.data(), + element_strides.data(), + interleave_mode, + swizzle_mode, + l2_promotion, + oob_fill + ); + if (ret_code != CUresult::CUDA_SUCCESS) { + auto print_vector = [&](auto t, const char* fmt, const char end='\n') { + for (auto elem : t) { + printf(fmt, elem); + } + printf("%c", end); + }; + fprintf(stderr, "Failed to create tensormap\n"); + fprintf(stderr, "Dim: %d\n", dim); + printf("size: "); print_vector(size, "%lu "); + printf("strides: "); print_vector(strides, "%lu "); + printf("box_size: "); print_vector(box_size, "%u "); + printf("element_strides: "); print_vector(element_strides, "%u "); + printf("global ptr: 0x%lx\n", (int64_t)global_ptr); + printf("data_type: %d\n", (int)data_type); + printf("swizzle_mode: %d\n", (int)swizzle_mode); + printf("l2_promotion: %d\n", (int)l2_promotion); + printf("interleave_mode: %d\n", (int)interleave_mode); + printf("oob_fill: %d\n", (int)oob_fill); + KU_ASSERT(false); + } + return result; +} + +// Given strides (in number of elements), this function converts their datatype in uint64_t and then multiplies by elem_size +template +static inline std::vector make_stride_helper(const std::vector &strides_in_elems, size_t elem_size) { + std::vector res; + for (auto stride : strides_in_elems) { + res.push_back(((uint64_t)stride) * elem_size); + } + return res; +} + +struct KernelLaunchConfig { + dim3 grid; + dim3 block; + size_t dynamic_smem; + cudaStream_t stream; + dim3 cluster{1, 1, 1}; + bool use_pdl{false}; + bool cooperative{false}; +}; + +namespace detail { + +template +void* kernel_arg_ptr(Arg&& arg) { + return const_cast(reinterpret_cast(std::addressof(arg))); +} + +} // namespace detail + +template +void launch_kernel(const KernelLaunchConfig &cfg, KernelFunc kernel, Args&&... args) { + if (cfg.dynamic_smem > 0) { + KU_CUDA_CHECK(cudaFuncSetAttribute( + reinterpret_cast(kernel), + cudaFuncAttributeMaxDynamicSharedMemorySize, + static_cast(cfg.dynamic_smem))); + } + + const bool is_cluster = !(cfg.cluster.x == 1 && cfg.cluster.y == 1 && cfg.cluster.z == 1); + const bool need_kernel_ex = is_cluster || cfg.use_pdl; + + if (is_cluster && cfg.cooperative) { + KU_ASSERT(false, "cluster and cooperative launch are mutually exclusive"); + } + + if (cfg.cooperative) { + void* kernel_args[sizeof...(Args) > 0 ? sizeof...(Args) : 1] = {}; + if constexpr (sizeof...(Args) > 0) { + size_t i = 0; + ((kernel_args[i++] = detail::kernel_arg_ptr(std::forward(args))), ...); + } + KU_CUDA_CHECK(cudaLaunchCooperativeKernel( + kernel, cfg.grid, cfg.block, + sizeof...(Args) > 0 ? kernel_args : nullptr, + static_cast(cfg.dynamic_smem), cfg.stream)); + } else if (need_kernel_ex) { + if (is_cluster) { + KU_ASSERT(cfg.grid.x % cfg.cluster.x == 0 && + cfg.grid.y % cfg.cluster.y == 0 && + cfg.grid.z % cfg.cluster.z == 0); + if (cfg.cluster.x * cfg.cluster.y * cfg.cluster.z > 8) { + KU_CUDA_CHECK(cudaFuncSetAttribute( + reinterpret_cast(kernel), + cudaFuncAttributeNonPortableClusterSizeAllowed, 1)); + } + } + + const unsigned int num_attrs = is_cluster ? 2 : 1; + cudaLaunchAttribute attrs[2]; + if (is_cluster) { + attrs[0].id = cudaLaunchAttributeClusterDimension; + attrs[0].val.clusterDim = {cfg.cluster.x, cfg.cluster.y, cfg.cluster.z}; + } + attrs[num_attrs - 1].id = cudaLaunchAttributeProgrammaticStreamSerialization; + attrs[num_attrs - 1].val.programmaticStreamSerializationAllowed = cfg.use_pdl ? 1 : 0; + + cudaLaunchConfig_t config = { + {cfg.grid.x, cfg.grid.y, cfg.grid.z}, + {cfg.block.x, cfg.block.y, cfg.block.z}, + cfg.dynamic_smem, + cfg.stream, + attrs, + num_attrs + }; + KU_CUDA_CHECK(cudaLaunchKernelEx( + &config, kernel, std::forward(args)...)); + } else { + kernel<<>>( + std::forward(args)...); + } + + KU_CHECK_KERNEL_LAUNCH(); +} + +#endif // KERUTILS_IS_BUILD_ON_CUDA + +} // namespace kerutils diff --git a/python/sglang/kernels/jit/csrc/deepselect/vendor/3rdparty/kerutils/include/kerutils/kerutils.cuh b/python/sglang/kernels/jit/csrc/deepselect/vendor/3rdparty/kerutils/include/kerutils/kerutils.cuh new file mode 100644 index 0000000000..3c511df98e --- /dev/null +++ b/python/sglang/kernels/jit/csrc/deepselect/vendor/3rdparty/kerutils/include/kerutils/kerutils.cuh @@ -0,0 +1,9 @@ +#pragma once + +#include "kerutils/common/common.h" + +#include "host/host.h" + +#ifdef KERUTILS_IS_BUILD_ON_CUDA +#include "device/device.cuh" +#endif diff --git a/python/sglang/kernels/jit/csrc/deepselect/vendor/LICENSE b/python/sglang/kernels/jit/csrc/deepselect/vendor/LICENSE new file mode 100644 index 0000000000..5c48bdc9f6 --- /dev/null +++ b/python/sglang/kernels/jit/csrc/deepselect/vendor/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 DeepSeek + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/python/sglang/kernels/jit/csrc/deepselect/vendor/cuda_kernels/bit_utils.cuh b/python/sglang/kernels/jit/csrc/deepselect/vendor/cuda_kernels/bit_utils.cuh new file mode 100644 index 0000000000..8bb0b0072c --- /dev/null +++ b/python/sglang/kernels/jit/csrc/deepselect/vendor/cuda_kernels/bit_utils.cuh @@ -0,0 +1,90 @@ +#pragma once + +#include +#include + +namespace topk_select_common { + +__device__ __forceinline__ +uint32_t bf16x2_to_u32(const nv_bfloat162 &v) { + static_assert(sizeof(nv_bfloat162) == sizeof(uint32_t)); + return *reinterpret_cast(&v); +} + +__device__ __forceinline__ +nv_bfloat162 u32_to_bf16x2(uint32_t u) { + static_assert(sizeof(nv_bfloat162) == sizeof(uint32_t)); + nv_bfloat162 v; + *reinterpret_cast(&v) = u; + return v; +} + +// Distort and un-distort: Map IEEE 754 floating-point total order to unsigned integer order. +// IEEE 754 bit representation (sign-magnitude with biased exponent and mantissa) does not +// preserve value order when interpreted as unsigned integer, and distort() fixes this: +// - For positive values (sign bit = 0): flip only the sign bit, mapping all positives to [0x80..., 0xFF...] +// - For negative values (sign bit = 1): flip all bits (~), inverting the magnitude order so that +// more negative (smaller value) => smaller uint +// After distortion, all negatives (0x00... ~ 0x7F...) < all positives (0x80... ~ 0xFF...) in +// unsigned integer comparison, which matches the floating-point total order. +// +// un_distort() is the exact inverse: flips the sign bit back for positives, flips all bits back for negatives. +template +__device__ __forceinline__ +UIntValueT distort(const UIntValueT &x) { + static_assert(sizeof(UIntValueT) == 2 || sizeof(UIntValueT) == 4); + if constexpr (sizeof(UIntValueT) == 2) { + UIntValueT mask = (x&0x8000) ? 0xFFFF : 0x8000; + return x ^ mask; + } else { + // mask = (x >> 31) | 0x80000000 is 0x80000000 for positives and 0xFFFFFFFF for negatives + uint32_t t, d; + asm ("shr.s32 %0, %2, 31;\n" + "lop3.b32 %1, %2, %0, 0x80000000, 0x1e;\n" + : "=r"(t), "=r"(d) + : "r"((uint32_t)x)); + return (UIntValueT)d; + } +} + +// A SIMD-like version of `distort`, which uses more efficient instructions for calculation +template +__device__ __forceinline__ +void distort_x2(UIntValueT result[2], const UIntValueT input[2]) { + static_assert(sizeof(UIntValueT) == 2 || sizeof(UIntValueT) == 4); + if constexpr (sizeof(UIntValueT) == 2) { + asm volatile ( + "{" + ".reg .u32 mask;" + "prmt.b32 mask, %1, 0, 0xbb99;" // mask[0:16] will be 0xFF if cur_value_packed[0]'s MSB is 1, 0x00 otherwise + "or.b32 mask, mask, 0x80008000;" + "xor.b32 %0, mask, %1;" + "}" + : "=r"(*(uint32_t*)(result)) + : "r"(*(uint32_t*)(input)) + ); + } else { + result[0] = distort(input[0]); + result[1] = distort(input[1]); + } +} + +template +__device__ __forceinline__ +UIntValueT un_distort(const UIntValueT &x) { + static_assert(sizeof(UIntValueT) == 2 || sizeof(UIntValueT) == 4); + if constexpr (sizeof(UIntValueT) == 2) { + UIntValueT mask = (x&0x8000) ? 0x8000 : 0xFFFF; + return x ^ mask; + } else { + // mask = ~(x >> 31) | 0x80000000 is 0xFFFFFFFF for positives and 0x80000000 for negatives + uint32_t t, d; + asm ("shr.s32 %0, %2, 31;\n" + "lop3.b32 %1, %2, %0, 0x80000000, 0x4b;\n" + : "=r"(t), "=r"(d) + : "r"((uint32_t)x)); + return (UIntValueT)d; + } +} + +} // namespace topk_select_common diff --git a/python/sglang/kernels/jit/csrc/deepselect/vendor/cuda_kernels/common_parts.cuh b/python/sglang/kernels/jit/csrc/deepselect/vendor/cuda_kernels/common_parts.cuh new file mode 100644 index 0000000000..819214e38b --- /dev/null +++ b/python/sglang/kernels/jit/csrc/deepselect/vendor/cuda_kernels/common_parts.cuh @@ -0,0 +1,1517 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include "structs.h" +#include "utils.cuh" +#include "cuda_kernels/config.h" +#include "cuda_kernels/bit_utils.cuh" + +using transac_bar_t = kerutils::transac_bar_t; + +namespace topk_select_common { + +#ifdef KERUTILS_ENABLE_SM100 + #define IS_LDG_STG_256_AVAILABLE 1 +#else + #define IS_LDG_STG_256_AVAILABLE 0 +#endif + +// Choose the loading & storing instruction +#if IS_LDG_STG_256_AVAILABLE + // Use LDG256 / STG256 on sm100+ + static constexpr uint32_t NUM_BYTES_PER_GMEM_LOAD = 256 / 8; + static constexpr uint32_t NUM_BYTES_PER_GMEM_STORE = 256 / 8; + #define LOAD_FROM_GMEM(src_ptr, dst_reg) KU_LDG_256(src_ptr, dst_reg, ".nc", "no_allocate", "evict_last", "256B") + #define STORE_TO_GMEM(dst_ptr, src_reg) KU_STG_256(dst_ptr, src_reg, "no_allocate", "evict_first") +#else + // Use LDG128 on sm90 and below + static constexpr uint32_t NUM_BYTES_PER_GMEM_LOAD = 128 / 8; + static constexpr uint32_t NUM_BYTES_PER_GMEM_STORE = 128 / 8; + #define LOAD_FROM_GMEM(src_ptr, dst_reg) KU_LDG_128(src_ptr, dst_reg, ".nc", "no_allocate", "256B") + #define STORE_TO_GMEM(dst_ptr, src_reg) KU_STG_128(dst_ptr, src_reg, "no_allocate", "") +#endif + +static constexpr uint32_t NUM_BYTES_PER_SMEM_LOAD = 16; +static constexpr uint32_t NUM_BYTES_PER_SMEM_STORE = 16; + + +template +__device__ __forceinline__ +void ld_shared(ValueT res[NUM_VALUES], const ValueT* ptr) { + constexpr uint32_t NUM_BYTES_TO_LOAD = NUM_VALUES * sizeof(ValueT); + static_assert(NUM_BYTES_TO_LOAD == 16 || NUM_BYTES_TO_LOAD == 8 || NUM_BYTES_TO_LOAD == 4 || NUM_BYTES_TO_LOAD == 2); + uint32_t addr = cute::cast_smem_ptr_to_uint(ptr); + if constexpr (NUM_BYTES_TO_LOAD == 16) { + *(__int128_t*)res = ku::ld_shared(ptr); + } else if constexpr (NUM_BYTES_TO_LOAD == 8) { + asm volatile ("ld.weak.shared::cta.b64 %0, [%1];\n" : "=l"(*(int64_t*)res) : "r"(addr) : "memory"); + } else if constexpr (NUM_BYTES_TO_LOAD == 4) { + asm volatile ("ld.weak.shared::cta.b32 %0, [%1];\n" : "=r"(*(int32_t*)res) : "r"(addr) : "memory"); + } else if constexpr (NUM_BYTES_TO_LOAD == 2) { + asm volatile ("ld.weak.shared::cta.b16 %0, [%1];\n" : "=h"(*(int16_t*)res) : "r"(addr) : "memory"); + } else { + __builtin_unreachable(); + } +} + +template +__device__ __forceinline__ +void ld_shared_with_loop(ValueT res[NUM_VALUES], const ValueT* ptr) { + constexpr uint32_t NUM_BYTES_TO_LOAD = NUM_VALUES * sizeof(ValueT); + if constexpr (NUM_BYTES_TO_LOAD <= 16) { + ld_shared(res, ptr); + } else { + static constexpr uint32_t NUM_LOAD_VALUES_THIS_ROUND = 16 / sizeof(ValueT); + static_assert(16 % sizeof(ValueT) == 0); + ld_shared(res, ptr); + ld_shared_with_loop(res+NUM_LOAD_VALUES_THIS_ROUND, ptr+NUM_LOAD_VALUES_THIS_ROUND); + } +} + +template +__device__ __forceinline__ +void st_global(ValueT* ptr, ValueT src[NUM_VALUES]) { + constexpr uint32_t NUM_BYTES_TO_STORE = NUM_VALUES * sizeof(ValueT); + static_assert((IS_LDG_STG_256_AVAILABLE && NUM_BYTES_TO_STORE == 32) || NUM_BYTES_TO_STORE == 16 || NUM_BYTES_TO_STORE == 8 || NUM_BYTES_TO_STORE == 4 || NUM_BYTES_TO_STORE == 2); + if constexpr (IS_LDG_STG_256_AVAILABLE && NUM_BYTES_TO_STORE == 32) { + KU_STG_256(ptr, src, "no_allocate", "evict_first"); + } else if constexpr (NUM_BYTES_TO_STORE == 16) { + KU_STG_128(ptr, src, "no_allocate", ""); + } else if constexpr (NUM_BYTES_TO_STORE == 8) { + *(uint64_t*)ptr = *(uint64_t*)src; + } else if constexpr (NUM_BYTES_TO_STORE == 4) { + *(uint32_t*)ptr = *(uint32_t*)src; + } else if constexpr (NUM_BYTES_TO_STORE == 2) { + *(uint16_t*)ptr = *(uint16_t*)src; + } else { + __builtin_unreachable(); + } +} + +template< + typename Config, + uint32_t MAX_TOPK, + uint32_t NUM_THREADS, + uint32_t TARGET_OCCUPANCY +> +struct EpilogueRunner { + using ValueT = typename Config::ValueT; + using OutIdxT = typename Config::OutIdxT; + static_assert(cute::is_same_v || cute::is_same_v); + static_assert(cute::is_same_v || cute::is_same_v); + using UIntValueT = cute::conditional_t, uint32_t, uint16_t>; + + static constexpr uint32_t NUM_WARPS = NUM_THREADS / 32; + static_assert(NUM_THREADS % 32 == 0); + static_assert(NUM_WARPS % 2 == 0); + + static constexpr uint32_t NUM_VALUES_PER_LOAD_STORE = NUM_BYTES_PER_GMEM_STORE / sizeof(ValueT); + static constexpr uint32_t NUM_VALUES_PER_THREAD_FOR_SORT = MAX_TOPK / NUM_THREADS; + static_assert(MAX_TOPK % NUM_THREADS == 0); + static constexpr uint32_t NUM_REGS_PER_THREAD = (65536 / (NUM_THREADS * TARGET_OCCUPANCY)) / 8 * 8; + using BlockRadixSortT = cub::BlockRadixSort< + UIntValueT, + NUM_THREADS, + NUM_VALUES_PER_THREAD_FOR_SORT, + uint32_t, + 4, + !(NUM_REGS_PER_THREAD <= 80 && NUM_VALUES_PER_THREAD_FOR_SORT >= 16) // MemoizeOuterScan. Setting this to `false` can reduce register pressure. We use a simple heuristic here + >; + // Gated on `sorted_value`: the sort only runs in the `sorted_value` branch, so this keeps the + // temp storage out of the smem plan of `sorted_value == false` kernels. + using BlockRadixSortTempStorageT = cute::conditional_t; + + template // "shortcut" means `end_vocab_idx` <= `topk` + static __device__ __forceinline__ void topk_select_epilogue( + typename Config::ValueT* smem_value_buf, // [MAX_TOPK] + uint32_t* smem_index_buf, // [MAX_TOPK] + const TopkSelectArgs &args, + uint32_t batch_idx, + uint32_t end_vocab_idx, + uint32_t warp_idx, + BlockRadixSortTempStorageT &radix_sort_temp_storage + ) { + // General epilogue for topk select + // This epilogue has two modes: shortcut mode and non-shortcut mode. + // In shortcut mode, it + // - Generates indices + // - Load values from global memory, if `Config::return_value` is `true` + // - Perform sorting, if `Config::sorted_value` is `true` + // - Save indices & values to global memory + // In non-shortcut mode, it + // - Assume end_vocab_idx > topk holds, i.e. there are enough values for topk selection + // - Assume topk indices locate at smem_index_buf[:topk] and topk values locate at smem_value_buf[:topk] (an extra `__syncthreads()` might be necessary) + // - Perform sorting, if `Config::sorted_value` is `true` + // - Save indices & values to global memory (values are un-distorted if `UNDISTORT_VALUES` is `true`) + + ValueT* input_values = Config::return_value ? (ValueT*)args.input + (uint64_t)batch_idx * args.stride_input_batch : nullptr; + OutIdxT* result_indices = (OutIdxT*)args.output_index + (uint64_t)batch_idx * args.stride_output_index_batch; + ValueT* result_values = Config::return_value ? (ValueT*)args.output_value + (uint64_t)batch_idx * args.stride_output_value_batch : nullptr; + int32_t output_idx_offset = args.output_idx_offset != nullptr ? __ldg(args.output_idx_offset + batch_idx) : 0; + ValueT oob_fill_value = Config::return_value && IS_SHORTCUT ? (ValueT)args.value_oob_fill_value : (ValueT)0.0f; + + if constexpr (!Config::sorted_value) { + if constexpr (IS_SHORTCUT) { + // Saves indices 0...end_vocab_idx (plus `output_idx_offset`) followed by (args.topk-end_vocab_idx) `args.idx_oob_fill_value` into `result_indices` + constexpr uint32_t NUM_OUTPUT_IDXS_PER_STORE = NUM_BYTES_PER_GMEM_STORE / sizeof(OutIdxT); + #pragma unroll 2 + for (uint32_t i = threadIdx.x * NUM_OUTPUT_IDXS_PER_STORE; i < args.topk; i += NUM_THREADS * NUM_OUTPUT_IDXS_PER_STORE) { + OutIdxT out[NUM_OUTPUT_IDXS_PER_STORE]; + CUTE_UNROLL + for (uint32_t j = 0; j < NUM_OUTPUT_IDXS_PER_STORE; ++j) + out[j] = i+j < end_vocab_idx ? (OutIdxT)(i+j) + output_idx_offset : args.idx_oob_fill_value; + STORE_TO_GMEM(result_indices + i, out); + } + + // Save values into `result_values`, if `RETURN_VALUE` is `True` + if constexpr (Config::return_value) { + #pragma unroll 2 + for (uint32_t i = threadIdx.x * NUM_VALUES_PER_LOAD_STORE; i < args.topk; i += NUM_THREADS * NUM_VALUES_PER_LOAD_STORE) { + ValueT values[NUM_VALUES_PER_LOAD_STORE]; + if (i < end_vocab_idx) { + LOAD_FROM_GMEM(input_values + i, values); + } + CUTE_UNROLL + for (uint32_t j = 0; j < NUM_VALUES_PER_LOAD_STORE; ++j) + values[j] = i+j < end_vocab_idx ? values[j] : oob_fill_value; + STORE_TO_GMEM(result_values + i, values); + } + } + } else { + // Load & cast & save indices + constexpr uint32_t NUM_OUTPUT_IDX_PER_ROUND = cute::min(NUM_BYTES_PER_SMEM_STORE / sizeof(uint32_t), NUM_BYTES_PER_GMEM_STORE / sizeof(OutIdxT)); + #pragma unroll 2 + for (uint32_t i = threadIdx.x * NUM_OUTPUT_IDX_PER_ROUND; i < args.topk; i += NUM_THREADS * NUM_OUTPUT_IDX_PER_ROUND) { + uint32_t indices_u32[NUM_OUTPUT_IDX_PER_ROUND]; + OutIdxT out[NUM_OUTPUT_IDX_PER_ROUND]; + ld_shared(indices_u32, smem_index_buf + i); + CUTE_UNROLL + for (uint32_t j = 0; j < NUM_OUTPUT_IDX_PER_ROUND; ++j) + out[j] = (OutIdxT)indices_u32[j] + output_idx_offset; + st_global(result_indices + i, out); + } + + if constexpr (Config::return_value) { + // Load & store values + constexpr uint32_t NUM_VALUES_PER_ROUND = cute::min(NUM_BYTES_PER_SMEM_STORE, NUM_BYTES_PER_GMEM_STORE) / sizeof(ValueT); + #pragma unroll 2 + for (uint32_t i = threadIdx.x * NUM_VALUES_PER_ROUND; i < args.topk; i += NUM_THREADS * NUM_VALUES_PER_ROUND) { + UIntValueT values[NUM_VALUES_PER_ROUND]; + ld_shared(values, (UIntValueT*)(smem_value_buf + i)); + st_global((UIntValueT*)(result_values + i), values); + } + } + } + } else { + // Load indices & values into shared memory + if constexpr (IS_SHORTCUT) { + if (warp_idx < NUM_WARPS/2) { + #pragma unroll 8 + for (uint32_t i = threadIdx.x; i < args.topk; i += NUM_THREADS/2) { + smem_index_buf[i] = i < end_vocab_idx ? i : args.idx_oob_fill_value - output_idx_offset; + } + } else { + constexpr uint32_t NUM_VALUES_PER_LDG128 = NUM_BYTES_PER_SMEM_LOAD / sizeof(ValueT); + #pragma unroll 2 + for (uint32_t i = (threadIdx.x-NUM_THREADS/2) * NUM_VALUES_PER_LDG128; i < end_vocab_idx; i += (NUM_THREADS/2) * NUM_VALUES_PER_LDG128) { + ValueT values[NUM_VALUES_PER_LDG128]; + KU_LDG_128(input_values + i, values, ".nc", "no_allocate", "256B"); + ku::st_shared(smem_value_buf + i, *(__int128_t*)values); + } + } + __syncthreads(); + } + + // Sort + UIntValueT local_values[NUM_VALUES_PER_THREAD_FOR_SORT]; + uint32_t local_indices[NUM_VALUES_PER_THREAD_FOR_SORT]; + uint32_t thread_offset = threadIdx.x * NUM_VALUES_PER_THREAD_FOR_SORT; + ld_shared_with_loop(local_values, (UIntValueT*)smem_value_buf + threadIdx.x * NUM_VALUES_PER_THREAD_FOR_SORT); + ld_shared_with_loop(local_indices, smem_index_buf + threadIdx.x * NUM_VALUES_PER_THREAD_FOR_SORT); + CUTE_UNROLL + for (uint32_t i = 0; i < NUM_VALUES_PER_THREAD_FOR_SORT; ++i) + local_values[i] = distort(local_values[i]); + if (IS_SHORTCUT || args.topk < MAX_TOPK) { + uint32_t limit = IS_SHORTCUT ? end_vocab_idx : args.topk; + CUTE_UNROLL + for (uint32_t i = 0; i < NUM_VALUES_PER_THREAD_FOR_SORT; ++i) + local_values[i] = thread_offset + i < limit ? local_values[i] : 0; + } + __syncthreads(); // To allow `radix_sort_temp_storage` overlap with `smem_value_buf` and `smem_index_buf` + BlockRadixSortT(radix_sort_temp_storage).SortDescending(local_values, local_indices); + + // Store directly to gmem (no smem buffering) + // For common cases, our MAX_TOPK <= 1024 and NUM_THREADS >= 256, so we have NUM_VALUES_PER_THREAD_FOR_SORT <= 8, which means each thread only need to call STG once. So we don't use SMEM to buffer here + if (thread_offset < args.topk) { + CUTE_UNROLL + for (uint32_t i = 0; i < NUM_VALUES_PER_THREAD_FOR_SORT; ++i) + local_values[i] = !IS_SHORTCUT || threadIdx.x * NUM_VALUES_PER_THREAD_FOR_SORT + i < end_vocab_idx ? un_distort(local_values[i]) : *(UIntValueT*)&oob_fill_value; + OutIdxT local_indices_new[NUM_VALUES_PER_THREAD_FOR_SORT]; + CUTE_UNROLL + for (uint32_t i = 0; i < NUM_VALUES_PER_THREAD_FOR_SORT; ++i) + local_indices_new[i] = (OutIdxT)((int32_t)local_indices[i] + output_idx_offset); + auto store = [&](T* dst, T src[NUM_VALUES]) { + constexpr uint32_t NUM_BYTES_TO_STORE = NUM_VALUES * sizeof(T); + if constexpr (NUM_BYTES_TO_STORE <= NUM_BYTES_PER_GMEM_STORE) { + // Don't need to do OOB check because the output array is at least padded to the maximum width of global store. + st_global(dst, src); + } else { + static_assert(NUM_BYTES_TO_STORE % NUM_BYTES_PER_GMEM_STORE == 0); + CUTE_UNROLL + for (uint32_t i = 0; i < NUM_BYTES_TO_STORE; i += NUM_BYTES_PER_GMEM_STORE) { + uint32_t elem_offset = i / sizeof(T); + if (thread_offset + elem_offset < args.topk) + STORE_TO_GMEM(dst+elem_offset, src+elem_offset); + } + } + }; + store.template operator()((UIntValueT*)result_values + thread_offset, local_values); + store.template operator()(result_indices + thread_offset, local_indices_new); + } + } + } +}; + +/* +// TODO Give a brief introduction to the algorithm here +*/ +template +class TopkSelectKernelBase { +public: + using ValueT = Config::ValueT; + using OutIdxT = Config::OutIdxT; + static_assert(std::is_same_v || std::is_same_v); + + static constexpr uint32_t PLACEHOLDER_U32 = std::is_same_v ? 0xff80 : 0xff800000; // -INF + static constexpr uint64_t PLACEHOLDER_B64 = std::is_same_v ? 0xff80ff80ff80ff80 : 0xff800000ff800000; + static constexpr uint64_t PLACEHOLDER_PAIR = (uint64_t)PLACEHOLDER_U32 << 32; + + static constexpr uint32_t TARGET_OCCUPANCY = Config::target_occupancy; + static constexpr uint32_t NUM_THREADS = Config::num_threads; + static constexpr uint32_t NUM_WARPS = NUM_THREADS / 32; + static constexpr uint32_t MAX_TOPK = Config::max_topk; + static_assert(NUM_THREADS == 256 || NUM_THREADS == 512); + + static constexpr uint32_t NUM_ELEMS_PER_128b = 16 / sizeof(ValueT); + static constexpr uint32_t NUM_UINT32_PER_128b = 4; + static constexpr uint32_t NUM_ELEMS_PER_ROUND = Config::elements_per_round; + static constexpr uint32_t NUM_ELEMS_PER_THREAD_PER_ROUND = NUM_ELEMS_PER_ROUND / NUM_THREADS; // TODO Unify "values", "elements", and "elems" to "elems" + static constexpr uint32_t NUM_128b_PER_THREAD_PER_ROUND = NUM_ELEMS_PER_THREAD_PER_ROUND / NUM_ELEMS_PER_128b; + static_assert(NUM_ELEMS_PER_ROUND % (NUM_ELEMS_PER_128b * NUM_THREADS) == 0); + static constexpr uint32_t ELEMS_PER_THREAD_PER_ROUND_MASK = NUM_ELEMS_PER_THREAD_PER_ROUND - 1; + // NUM_128b_PER_THREAD_PER_ROUND is the canonical name; no separate NUM_UNITS alias is kept. + + static_assert(NUM_ELEMS_PER_THREAD_PER_ROUND == 16); + + // Each segment/window contains this many elements. It is the granularity used by both the + // pseudo-random visit order and the sorted-index epilogue. + static constexpr uint32_t NUM_ELEMS_PER_SEG = Config::elements_per_segment; // shuffle rule + static_assert(NUM_ELEMS_PER_SEG == 512); + // The last NUM_TAIL_ELEMS elements of a row are always visited in their original order. + // 4096 keeps the tail small enough to fit alongside the init-window perm segments. + static constexpr uint32_t NUM_TAIL_ELEMS = 4096; + static constexpr uint32_t NUM_TAIL_SEGS = NUM_TAIL_ELEMS / NUM_ELEMS_PER_SEG; + // The init window is 32 KiB of input data + static constexpr uint32_t NUM_ELEMS_IN_INIT_WINDOW = 32768 / sizeof(ValueT); + static constexpr uint32_t NUM_INIT_ROUNDS_MAX = NUM_ELEMS_IN_INIT_WINDOW / NUM_ELEMS_PER_ROUND; + static_assert(NUM_ELEMS_IN_INIT_WINDOW % NUM_ELEMS_PER_ROUND == 0); + static_assert(NUM_ELEMS_IN_INIT_WINDOW % NUM_ELEMS_PER_SEG == 0); + + static constexpr uint32_t NUM_SEGS_PER_ROUND = NUM_ELEMS_PER_ROUND / NUM_ELEMS_PER_SEG; + static_assert(NUM_ELEMS_PER_ROUND % NUM_ELEMS_PER_SEG == 0); + static_assert(NUM_ELEMS_PER_SEG % NUM_ELEMS_PER_128b == 0); + static constexpr uint32_t NUM_128b_PER_SEG = NUM_ELEMS_PER_SEG / NUM_ELEMS_PER_128b; + static_assert(NUM_THREADS % NUM_128b_PER_SEG == 0); + static_assert(NUM_128b_PER_SEG % 32 == 0); // every lane in a warp should in the same seg + + static constexpr uint32_t NUM_ISSUE_WARPS = NUM_SEGS_PER_ROUND < 4 ? NUM_SEGS_PER_ROUND : 4; + static constexpr uint32_t NUM_SEGS_PER_ISSUE_WARP = NUM_SEGS_PER_ROUND / NUM_ISSUE_WARPS; + static_assert(NUM_SEGS_PER_ROUND % NUM_ISSUE_WARPS == 0); + static_assert(NUM_TAIL_SEGS % NUM_ISSUE_WARPS == 0); + static_assert(NUM_WARPS >= NUM_ISSUE_WARPS); + + static constexpr uint32_t NUM_TMA_LOAD_BUFS = Config::tma_buffer_depth; + static constexpr uint32_t TMA_PREFETCH_DEPTH = NUM_TMA_LOAD_BUFS - 1; + static_assert(NUM_TMA_LOAD_BUFS >= 2); + + static constexpr uint32_t RECONSTRUCT_THRESHOLD = Config::reconstruct_threshold; + static_assert(RECONSTRUCT_THRESHOLD >= MAX_TOPK); + + // Parameters of the pseudo-random permutation over permuted segments: + // permuted_id = (linear_id * (PERM_MUL_PRIME % perm_len) + PERM_ADD_BASE) % perm_len. + static constexpr uint32_t PERM_ADD_BASE = 0x22262226u; + static constexpr uint32_t PERM_MUL_PRIME = 0xB559EB75u; + + static constexpr uint32_t NUM_SEGS_IN_INIT_WINDOW = NUM_ELEMS_IN_INIT_WINDOW / NUM_ELEMS_PER_SEG; + // Number of permuted segments included in the init window after reserving room for the tail. + static constexpr uint32_t NUM_PERM_SEGS_IN_INIT_WINDOW = + (NUM_ELEMS_IN_INIT_WINDOW - NUM_TAIL_ELEMS) / NUM_ELEMS_PER_SEG; + static_assert(NUM_PERM_SEGS_IN_INIT_WINDOW % NUM_TAIL_SEGS == 0); + static_assert(NUM_PERM_SEGS_IN_INIT_WINDOW > 0); + static constexpr uint32_t NUM_UINT32_IN_INIT_WINDOW_PER_THREAD = NUM_ELEMS_IN_INIT_WINDOW * sizeof(ValueT) / sizeof(uint32_t) / NUM_THREADS; + static constexpr uint32_t NUM_128b_INIT_PER_THREAD = NUM_UINT32_IN_INIT_WINDOW_PER_THREAD / NUM_UINT32_PER_128b; + static_assert(NUM_ELEMS_IN_INIT_WINDOW % (sizeof(uint32_t) / sizeof(ValueT) * NUM_THREADS) == 0); + static_assert(NUM_UINT32_IN_INIT_WINDOW_PER_THREAD % (16 * sizeof(ValueT) / sizeof(uint32_t)) == 0); + static_assert(2 * NUM_UINT32_IN_INIT_WINDOW_PER_THREAD <= 256); + + static constexpr uint32_t NUM_EXTRA_SLOTS = RECONSTRUCT_THRESHOLD + NUM_ELEMS_PER_ROUND; + + static constexpr uint32_t NUM_RECONSTRUCT_RADIX_BITS = 8; + static constexpr uint32_t NUM_RECONSTRUCT_BUCKETS = 1u << NUM_RECONSTRUCT_RADIX_BITS; + // Each row also holds one sink slot (index NUM_RECONSTRUCT_BUCKETS) that the LSB histogram sends the + // out-of-bucket elements to; the +4 rounds a row up to a multiple of 16 bytes so that both rows stay + // 16B-aligned, since the reader relies on for its 128-bit shared-memory loads. + static constexpr uint32_t NUM_RECONSTRUCT_BUCKET_SLOTS = NUM_RECONSTRUCT_BUCKETS + 4; + static_assert(sizeof(uint32_t) * NUM_RECONSTRUCT_BUCKET_SLOTS % 16 == 0); + static_assert(MAX_TOPK + NUM_EXTRA_SLOTS <= 0xFFFF); + static_assert(NUM_EXTRA_SLOTS % 2 == 0); + + static constexpr uint32_t NUM_RECONSTRUCT_UNITS_MAX = (MAX_TOPK + NUM_EXTRA_SLOTS) / 2; + + using EpilogueT = EpilogueRunner; + static_assert(sizeof(typename EpilogueT::BlockRadixSortTempStorageT) <= NUM_EXTRA_SLOTS * sizeof(uint64_t)); + + static_assert(NUM_EXTRA_SLOTS * sizeof(uint64_t) >= NUM_ELEMS_IN_INIT_WINDOW * sizeof(ValueT), + "the init window must fit in the extra pairs region"); + static_assert(NUM_EXTRA_SLOTS * sizeof(uint64_t) / sizeof(uint32_t) >= MAX_VOCAB_SIZE / NUM_ELEMS_PER_SEG, + "the index sort's window delta table must fit in the extra pairs region"); + // array `incoming_topk_pairs` will be used as + // 1. tma buffer during init phase + // 2. extra array during main phase + // 3. windows_delta counter during epilogue phase + static_assert(2 * MAX_TOPK * sizeof(uint64_t) % 1024 == 0); + + static constexpr uint32_t SWIZZLE_SHIFT = 3; + static constexpr uint32_t SWIZZLE_MASK = 7 * (16 / sizeof(ValueT)); + + static constexpr uint32_t NUM_BYTES_PER_TMA_ROW = 128; // Limit the innermost box dim of TMA to 1) prevent OOB 2) swizzling has limitations on the innermost box dim + static_assert(NUM_BYTES_PER_TMA_ROW <= INPUT_STRIDE_ALIGNMENT_REQUIREMENT); // To prevent OOB + static constexpr uint32_t NUM_ELEMS_PER_TMA_ROW = NUM_BYTES_PER_TMA_ROW / sizeof(ValueT); + static_assert(NUM_ELEMS_PER_SEG % NUM_ELEMS_PER_TMA_ROW == 0); + static constexpr uint32_t NUM_TMA_ROWS_PER_SEG = NUM_ELEMS_PER_SEG / NUM_ELEMS_PER_TMA_ROW; + static_assert(SWIZZLE_MASK == ((1u << SWIZZLE_SHIFT) - 1) * NUM_ELEMS_PER_128b); + static_assert(NUM_ELEMS_PER_TMA_ROW == (1u << SWIZZLE_SHIFT) * NUM_ELEMS_PER_128b); + static_assert(NUM_TMA_ROWS_PER_SEG % (1u << SWIZZLE_SHIFT) == 0); + + static_assert(NUM_SEGS_PER_ROUND == NUM_WARPS); + static_assert(NUM_SEGS_PER_ROUND >= NUM_TAIL_SEGS); + static_assert(NUM_ELEMS_PER_SEG % (Config::elements_per_round/NUM_WARPS) == 0); + + struct TmaParams { + CUtensorMap tensor_map; + }; + + static CUtensorMap make_topk_tensor_map(const TopkSelectArgs &args) { + static_assert(INPUT_STRIDE_ALIGNMENT_REQUIREMENT >= NUM_ELEMS_PER_TMA_ROW * sizeof(ValueT)); + constexpr CUtensorMapDataType dtype = std::is_same_v + ? CU_TENSOR_MAP_DATA_TYPE_BFLOAT16 + : CU_TENSOR_MAP_DATA_TYPE_FLOAT32; + return ku::make_tensor_map( + // Split args.vocab_size into two dims because + // - TMA requires innermost box dim <= swizzle size + // - To avoid OOB as we have `INPUT_STRIDE_ALIGNMENT_REQUIREMENT` + { + NUM_ELEMS_PER_TMA_ROW, + ku::ceil_div((uint64_t)args.vocab_size, (uint64_t)NUM_ELEMS_PER_TMA_ROW), + args.batch_size + }, + ku::make_stride_helper({NUM_ELEMS_PER_TMA_ROW, args.stride_input_batch}, sizeof(ValueT)), + { + NUM_ELEMS_PER_TMA_ROW, + NUM_TMA_ROWS_PER_SEG, + 1 + }, + args.input, + dtype, + CUtensorMapSwizzle::CU_TENSOR_MAP_SWIZZLE_128B, + CUtensorMapL2promotion::CU_TENSOR_MAP_L2_PROMOTION_L2_256B + ); + } + + struct SharedMemoryPlanBase { + // bf16: pair(64b) = unused(16b) | value(16b) | index(32b) + // fp32: pair(64b) = value(32b) | index(32b) + // candidate[0]/candidate[1] are the A/B buffers swapped on each reconstruct; + // incoming holds the pairs scanned since the last reconstruct. + CUTE_ALIGNAS(1024) uint64_t surviving_topk_pairs[2][MAX_TOPK]; + CUTE_ALIGNAS(1024) uint64_t incoming_topk_pairs[NUM_EXTRA_SLOTS]; + CUTE_ALIGNAS(1024) ValueT tma_load_buf[NUM_TMA_LOAD_BUFS][NUM_ELEMS_PER_ROUND]; + transac_bar_t tma_load_full_bar[NUM_TMA_LOAD_BUFS]; + transac_bar_t init_full_bar[NUM_INIT_ROUNDS_MAX]; + uint32_t warp_cnt[NUM_WARPS]; + uint32_t reconstruct_pivot_bucket; + uint32_t reconstruct_num_should_select; + CUTE_ALIGNAS(16) uint32_t reconstruct_bucket_counter[2][NUM_RECONSTRUCT_BUCKET_SLOTS]; + }; + + // Map a linear visit index to the actual permuted segment id. + static __device__ __forceinline__ + uint32_t get_permuted_seg_idx(uint32_t linear_id, uint32_t perm_len, uint32_t perm_mul) { + return ((linear_id % perm_len) * perm_mul + PERM_ADD_BASE) % perm_len; + } + + static __device__ __forceinline__ + void advance_perm_state(uint32_t &state, uint32_t stride, uint32_t perm_len) { + state += stride; + if (state >= perm_len) state -= perm_len; + } + + static __device__ __forceinline__ + uint32_t sw_msk(uint32_t e) { + return (e >> SWIZZLE_SHIFT) & SWIZZLE_MASK; + } + static __device__ __forceinline__ + uint32_t sw_elem(uint32_t e) { + return e ^ sw_msk(e); + } + static __device__ __forceinline__ + uint32_t sw_b128(uint32_t u) { + return u ^ ((u >> 3) & (7)); + } + + struct EqGtPrefix { uint32_t start_pos_in_collector; uint32_t eq_quota; }; + // Given local > pivot and == pivot counts, compute this thread's output prefix (start position in the final output buffer) and how many equal + // elements it may still append (eq_quota). + // Should be called by all threads + // cnt_gt and cnt_eq must be < 65536. + static __device__ __forceinline__ + EqGtPrefix compute_equal_quota_and_prefix(uint32_t cnt_gt, uint32_t cnt_eq, uint32_t topk, uint32_t warp_idx, uint32_t lane_idx, uint32_t *warp_cnt) { + static_assert(NUM_WARPS <= 32); + uint32_t cnt_packed = (cnt_gt << 16) | cnt_eq; + uint32_t warp_total_packed = __reduce_add_sync(0xFFFFFFFF, cnt_packed); + if (lane_idx == 0) { + warp_cnt[warp_idx] = warp_total_packed; + } + __syncthreads(); + + uint32_t off_lane_packed = warp_level_exclusive_prefix_sum(cnt_packed, lane_idx); + uint32_t stored_warp_packed = lane_idx < NUM_WARPS ? warp_cnt[lane_idx] : 0u; + uint32_t off_warp_packed = __reduce_add_sync(0xFFFFFFFF, lane_idx < warp_idx ? stored_warp_packed : 0u); + uint32_t total_packed = __reduce_add_sync(0xFFFFFFFF, stored_warp_packed); + uint32_t num_total_ge = total_packed >> 16; + uint32_t num_total_eq_quota = topk - num_total_ge; + uint32_t off_packed = off_warp_packed + off_lane_packed; + uint32_t num_gt_before = off_packed >> 16; + uint32_t num_eq_before = off_packed & 0xFFFFu; + uint32_t eq_quota = num_total_eq_quota > num_eq_before ? num_total_eq_quota - num_eq_before : 0u; + uint32_t num_elems_should_select_before = num_gt_before + min(num_eq_before, num_total_eq_quota); + return {num_elems_should_select_before, eq_quota}; + } + + // Load this thread's slice from the swizzled smem round buffer into registers. + template + static __device__ __forceinline__ + void load_swizzled_slice( + ValueT dst[NUM_128b * NUM_ELEMS_PER_128b], + const ValueT *smem_buf, + uint32_t smem_slice_elem_offset, // element offset into smem_buf where this thread's slice starts + uint32_t unit_swizzle_mask, // swizzle mask applied to each 128-bit unit inside the slice + uint32_t first_128b_unit_in_slice = 0 // which 128-bit unit of this slice to start loading from + ) { + CUTE_UNROLL + for (uint32_t u = 0; u < NUM_128b; ++u) { + ld_shared( + dst + u * NUM_ELEMS_PER_128b, + smem_buf + smem_slice_elem_offset + + (((first_128b_unit_in_slice + u) * NUM_ELEMS_PER_128b) ^ unit_swizzle_mask)); + } + } + + // Find the bucket id that the topk-th element resides in, and get how many elements we should select from that bucket + // Save result to `smem.reconstruct_pivot_bucket` and `smem.reconstruct_num_should_select` + // If `CHECK_IF_SHOULD_SELECT_WHOLE_BUCKET`, return whether we should select all elements in that bucket. + // Should be called by a whole warp + template + static __device__ __forceinline__ + bool find_pivot_in_histogram(SharedMemoryPlanBase &smem, const uint32_t *bucket_counter, uint32_t topk, uint32_t lane_idx) { + bool should_select_whole_bucket = false; + // Each lane loads 8 counters + static_assert(NUM_RECONSTRUCT_BUCKETS == 32 * 8); + uint32_t counts[8]; + const uint32_t *bucket_ptr = bucket_counter + lane_idx * 8; + ld_shared<4>(counts, bucket_ptr); + ld_shared<4>(counts + 4, bucket_ptr + 4); + + uint32_t local_sum = 0; + CUTE_UNROLL + for (uint32_t j = 0; j < 8; ++j) + local_sum += counts[j]; + + // Get the inclusive suffix sum of the histogram + uint32_t suffix_count[9]; + suffix_count[8] = warp_level_exclusive_suffix_sum(local_sum, lane_idx); + CUTE_UNROLL + for (int32_t j = 7; j >= 0; --j) + suffix_count[j] = suffix_count[j + 1] + counts[j]; + + if (suffix_count[8] < topk && topk <= suffix_count[0]) { + uint32_t j = 0; + CUTE_UNROLL + for (uint32_t k = 1; k < 8; ++k) + j += (uint32_t)(suffix_count[k] >= topk); + + // A depth-3 SEL tree picking suffix_count[j+1] by the bits of j + // j is the bucket that contains the top-k element + uint32_t b0 = (j & 1) ? suffix_count[2] : suffix_count[1], b1 = (j & 1) ? suffix_count[4] : suffix_count[3]; + uint32_t b2 = (j & 1) ? suffix_count[6] : suffix_count[5], b3 = (j & 1) ? suffix_count[8] : suffix_count[7]; + b0 = (j & 2) ? b1 : b0; b2 = (j & 2) ? b3 : b2; + uint32_t suffix_count_j_plus_1 = (j & 4) ? b2 : b0; // = s[j+1] + smem.reconstruct_pivot_bucket = lane_idx * 8 + j; + smem.reconstruct_num_should_select = topk - suffix_count_j_plus_1; + + if constexpr (CHECK_IF_SHOULD_SELECT_WHOLE_BUCKET) { + // topk == suffix_count[j] means the whole pivot bucket is selected + uint32_t d0 = (j & 1) ? suffix_count[1] : suffix_count[0], d1 = (j & 1) ? suffix_count[3] : suffix_count[2]; + uint32_t d2 = (j & 1) ? suffix_count[5] : suffix_count[4], d3 = (j & 1) ? suffix_count[7] : suffix_count[6]; + d0 = (j & 2) ? d1 : d0; d2 = (j & 2) ? d3 : d2; + uint32_t suffix_count_j = (j & 4) ? d2 : d0; // = suffix_count[j] + should_select_whole_bucket = (topk == suffix_count_j); + } + } + + if constexpr (CHECK_IF_SHOULD_SELECT_WHOLE_BUCKET) { + // The owning lane holds the result; reduce so all lanes in the warp see the same answer. + return __reduce_or_sync(0xFFFFFFFFu, should_select_whole_bucket); + } + return false; + } + + // Clean every bucket counter using 16-byte shared-memory writes. + // Need to be called by all threads. + static __device__ __forceinline__ + void clear_reconstruct_histograms(SharedMemoryPlanBase &smem, uint32_t thread_idx) { + static_assert((2 * NUM_RECONSTRUCT_BUCKET_SLOTS) % 4 == 0); + __int128_t zero = (__int128_t)0ull; + for (uint32_t i = thread_idx; i < (2 * NUM_RECONSTRUCT_BUCKET_SLOTS) / 4; i += NUM_THREADS) { + reinterpret_cast<__int128_t*>(smem.reconstruct_bucket_counter)[i] = zero; + } + } + + template + static __device__ __forceinline__ + void init_shared_memory(SharedMemoryPlanBase &smem, uint32_t warp_idx, ExtraBarInitF &&init_extra_bar_func) { + // Pad unused elements in `surviving_topk_pairs` as `-INF` to avoid being selected + static_assert((2 * MAX_TOPK) % NUM_THREADS == 0); + CUTE_UNROLL + for (uint32_t i = 0; i < MAX_TOPK / NUM_THREADS; ++i) { + smem.surviving_topk_pairs[0][i * NUM_THREADS + threadIdx.x] = PLACEHOLDER_PAIR; + smem.surviving_topk_pairs[1][i * NUM_THREADS + threadIdx.x] = PLACEHOLDER_PAIR; + } + + clear_reconstruct_histograms(smem, threadIdx.x); + + if (warp_idx == 0 && cute::elect_one_sync()) { + CUTE_UNROLL + for (uint32_t i = 0; i < NUM_TMA_LOAD_BUFS; ++i) { + smem.tma_load_full_bar[i].init(NUM_ISSUE_WARPS); + } + CUTE_UNROLL + for (uint32_t i = 0; i < NUM_INIT_ROUNDS_MAX; ++i) { + smem.init_full_bar[i].init(NUM_ISSUE_WARPS); + } + init_extra_bar_func(); + cutlass::arch::fence_barrier_init(); + } + __syncthreads(); + } + + static __device__ __forceinline__ + void init_shared_memory(SharedMemoryPlanBase &smem, uint32_t warp_idx) { + init_shared_memory(smem, warp_idx, [] {}); + } + + // Issue TMA loads for one round + // Use IS_INIT to control whether is the init round. + // For init round: copy the "tail" [num_perm_segs * NUM_ELEMS_PER_SEG, end_vocab_idx), if HAVE_TAIL is true, and also the first NUM_SEGS_PER_ROUND - NUM_TAIL_SEGS segments according to the perm generator + // For non-init round: copy the next NUM_SEGS_PER_ROUND segs according to the perm generator + template + static __device__ __forceinline__ + void issue_tma_loads_for_round(SharedMemoryPlanBase &smem, const CUtensorMap &tensor_map, + uint32_t batch_idx, uint32_t end_vocab_idx, + uint32_t num_perm_segs, uint32_t round_idx, + uint32_t &next_tma_permuted_segment, uint32_t tma_permuted_segment_stride, + uint32_t warp_idx + ) { + uint32_t perm_len = max(num_perm_segs, 1u); + uint32_t num_perm_elems = num_perm_segs * NUM_ELEMS_PER_SEG; + + static_assert(IS_INIT || !HAVE_TAIL); + if (warp_idx >= NUM_ISSUE_WARPS) return; + + ValueT *dst_base; + ku::transac_bar_t *bar_ptr; + if constexpr (IS_INIT) { + // Use `smem.incoming_topk_pairs` as a buffer for elements in the "init" rounds + // We've asserted the size of `incoming_topk_pairs` is large enough + dst_base = (ValueT*)smem.incoming_topk_pairs + round_idx * NUM_ELEMS_PER_ROUND; + bar_ptr = &smem.init_full_bar[round_idx]; + } else { + dst_base = smem.tma_load_buf[round_idx % NUM_TMA_LOAD_BUFS]; + bar_ptr = &smem.tma_load_full_bar[round_idx % NUM_TMA_LOAD_BUFS]; + } + ku::transac_bar_t &bar = *bar_ptr; + + constexpr uint32_t NUM_BYTES_PER_SEG = NUM_ELEMS_PER_SEG * (uint32_t)sizeof(ValueT); + auto copy_one_seg = [&](uint32_t dst_seg_idx, uint32_t global_seg_idx) { + cute::SM90_TMA_LOAD_3D::copy( + &tensor_map, reinterpret_cast(&bar), + (uint64_t)cute::TMA::CacheHintSm90::EVICT_FIRST, + dst_base + dst_seg_idx * NUM_ELEMS_PER_SEG, + 0, global_seg_idx * NUM_TMA_ROWS_PER_SEG, batch_idx); + }; + + if (IS_INIT && num_perm_segs == 0) { + // Do not permute if `num_perm_seg == 0`, just copy in all the elements in-a-row + uint32_t num_global_segs = ku::ceil_div(end_vocab_idx, (uint32_t)NUM_ELEMS_PER_SEG); + uint32_t num_segs_issued = 0; + CUTE_UNROLL + for (uint32_t k = 0; k < NUM_SEGS_PER_ISSUE_WARP; k++) { + uint32_t local_seg_idx = warp_idx + k * NUM_ISSUE_WARPS; + uint32_t global_seg_idx = round_idx * NUM_SEGS_PER_ROUND + local_seg_idx; + if (global_seg_idx < num_global_segs) { + copy_one_seg(local_seg_idx, global_seg_idx); + num_segs_issued += 1; + } + } + bar.arrive_and_expect_tx(num_segs_issued * NUM_BYTES_PER_SEG); + } else { + static_assert(NUM_TAIL_SEGS % NUM_ISSUE_WARPS == 0); + constexpr uint32_t NUM_TAIL_SEGS_PER_ISSUE_WARP = HAVE_TAIL ? NUM_TAIL_SEGS / NUM_ISSUE_WARPS : 0; + if constexpr (HAVE_TAIL) { + // Copy tail segs + // To save memory BW, only copy segs that have at least one valid element. The other segs are handled by `fill_padded_tail_segments` + uint32_t num_tail_tma_segs = ku::ceil_div(end_vocab_idx - num_perm_elems, (uint32_t)NUM_ELEMS_PER_SEG); + uint32_t num_real_tail_segs_issued = 0; + CUTE_UNROLL + for (uint32_t i = 0; i < NUM_TAIL_SEGS_PER_ISSUE_WARP; i++) { + uint32_t cur_seg_idx = warp_idx + i * NUM_ISSUE_WARPS; + if (cur_seg_idx < num_tail_tma_segs) { + copy_one_seg(cur_seg_idx, (num_perm_segs + cur_seg_idx)); + num_real_tail_segs_issued += 1; + } + } + bar.arrive_and_expect_tx((num_real_tail_segs_issued + NUM_SEGS_PER_ISSUE_WARP - NUM_TAIL_SEGS_PER_ISSUE_WARP) * NUM_BYTES_PER_SEG); + } else { + bar.arrive_and_expect_tx(NUM_SEGS_PER_ISSUE_WARP * NUM_BYTES_PER_SEG); + } + CUTE_UNROLL + for (uint32_t i = NUM_TAIL_SEGS_PER_ISSUE_WARP; i < NUM_SEGS_PER_ISSUE_WARP; i++) { + uint32_t cur_global_seg_idx = next_tma_permuted_segment; + uint32_t local_seg_idx = warp_idx + i * NUM_ISSUE_WARPS; + copy_one_seg(local_seg_idx, cur_global_seg_idx); + advance_perm_state(next_tma_permuted_segment, tma_permuted_segment_stride, perm_len); + } + } + } + + // Fill the invalid (padded) tail segments with PLACEHOLDER (-INF). + // TMA only loads the real tail elements; the unused padded tail region must still be initialized as -INF + static __device__ __forceinline__ void fill_padded_tail_segments(ValueT *init_buf, uint32_t num_tail_elems) { + uint32_t tail_covered_by_tma = ku::ceil(num_tail_elems, (uint32_t)NUM_ELEMS_PER_SEG); + constexpr __int128_t FILLER_128b = ((__int128_t)PLACEHOLDER_B64 << 64) | PLACEHOLDER_B64; + for (uint32_t i = tail_covered_by_tma + threadIdx.x * NUM_ELEMS_PER_128b; i < NUM_TAIL_ELEMS; i += NUM_THREADS * NUM_ELEMS_PER_128b) { + ku::st_shared(init_buf + i, FILLER_128b); + } + } + + // Stage the final top-k candidates from the survivor buffer into a contiguous smem layout + // and run the epilogue. + // + // When SORT_BY_INDEX is true, each 512-element window is visited as a whole in the permuted + // order, so the survivors are already sorted inside each window. Only the window groups need + // to be reordered by their window id + // + // Input cannot contain NaN + template + static __device__ __forceinline__ + void stage_output_and_epilogue(SharedMemoryPlanBase &smem, typename EpilogueT::BlockRadixSortTempStorageT &sort_temp_storage, + const TopkSelectArgs &args, uint32_t batch_idx, uint32_t end_vocab_idx, + uint32_t survivor_buf_idx, uint32_t warp_idx, uint32_t lane_idx) { + + uint32_t staging_buf_idx = survivor_buf_idx ^ 1; + bool is_already_sorted_in_index = end_vocab_idx <= NUM_ELEMS_IN_INIT_WINDOW; + uint32_t* window_scatter_delta = (uint32_t*)smem.incoming_topk_pairs; + ValueT* smem_value_buf = (ValueT*)smem.surviving_topk_pairs[staging_buf_idx]; + uint32_t* smem_index_buf = (uint32_t*)(smem.surviving_topk_pairs[staging_buf_idx] + MAX_TOPK * (uint32_t)sizeof(ValueT) / 8); + + // Index-sort shuffle: each window's selected candidates are already sorted and contiguous; + // only the whole window groups are in visit order, so reorder them by their window id. + if constexpr (SORT_BY_INDEX) if (!is_already_sorted_in_index) { + uint32_t num_windows = ku::ceil_div(end_vocab_idx, (uint32_t)NUM_ELEMS_PER_SEG); + auto get_window_id = [&](uint32_t offset) -> uint32_t { + return (uint32_t)smem.surviving_topk_pairs[survivor_buf_idx][offset] / NUM_ELEMS_PER_SEG; + }; + + // Reset all window counters. + for (uint32_t i = threadIdx.x; i < num_windows; i += NUM_THREADS) { + window_scatter_delta[i] = 0; + } + __syncthreads(); + + // Count how many selected candidates fall into each window. + for (uint32_t i = threadIdx.x; i < args.topk; i += NUM_THREADS) { + atomicAdd_block(window_scatter_delta + get_window_id(i), 1u); + } + __syncthreads(); + + // Convert the per-window counts into exclusive prefix sums (starting offsets). + uint32_t windows_per_thread = ku::ceil_div(num_windows, (uint32_t)NUM_THREADS); + uint32_t w_lo = min(threadIdx.x * windows_per_thread, num_windows); + uint32_t w_hi = min(w_lo + windows_per_thread, num_windows); + uint32_t my_cnt = 0; + for (uint32_t i = w_lo; i < w_hi; ++i) { + // TODO Performance can be optimized by using wider reads (pay attn to alignment issues!) + my_cnt += window_scatter_delta[i]; + } + uint32_t warp_total = __reduce_add_sync(0xFFFFFFFF, my_cnt); + if (lane_idx == 0) { + smem.warp_cnt[warp_idx] = warp_total; + } + __syncthreads(); + + static_assert(NUM_WARPS <= 32); + uint32_t cur_prefix_sum = __reduce_add_sync(0xFFFFFFFF, lane_idx < warp_idx ? smem.warp_cnt[lane_idx] : 0u) + warp_level_exclusive_prefix_sum(my_cnt, lane_idx); + for (uint32_t i = w_lo; i < w_hi; ++i) { + uint32_t cnt = window_scatter_delta[i]; + window_scatter_delta[i] = cur_prefix_sum; + cur_prefix_sum += cnt; + } + __syncthreads(); + + // Subtract window_delta[i] by the index of the first element in this window around all select elements + // So that later we can use "index + window_delta[i]" to choose the index + for (uint32_t i = threadIdx.x+1; i < args.topk; i += NUM_THREADS) { // +1 to skip the first element + uint32_t idx = (uint32_t)smem.surviving_topk_pairs[survivor_buf_idx][i]; + uint32_t window_idx = idx / NUM_ELEMS_PER_SEG; + uint32_t prev_elem_window_id = get_window_id(i - 1u); + if (window_idx != prev_elem_window_id) { + // i is the first element in its window + window_scatter_delta[window_idx] -= i; + } + } + + __syncthreads(); + } + + CUTE_UNROLL + for (uint32_t i = threadIdx.x; i < MAX_TOPK; i += NUM_THREADS) { + uint64_t pair = smem.surviving_topk_pairs[survivor_buf_idx][i]; + uint32_t idx = (uint32_t)pair; + ValueT val; + if constexpr (sizeof(ValueT) == 2) { + val = __ushort_as_bfloat16((uint16_t)(pair >> 32)); + } else { + val = __uint_as_float((uint32_t)(pair >> 32)); + } + if (SORT_BY_INDEX && !is_already_sorted_in_index) { + if (i < args.topk) { + uint32_t dst_pos = (i + window_scatter_delta[idx / NUM_ELEMS_PER_SEG]) & (MAX_TOPK - 1); + smem_value_buf[dst_pos] = val; + smem_index_buf[dst_pos] = idx; + continue; + } + } + smem_value_buf[i] = val; + smem_index_buf[i] = idx; + } + + __syncthreads(); + + EpilogueT::template topk_select_epilogue( + smem_value_buf, + smem_index_buf, + args, + batch_idx, end_vocab_idx, warp_idx, + sort_temp_storage + ); + } + + // Take the appropriate action when we see a NaN (i.e. `trap()` or write 0x3f3f3f3f) + static __device__ __forceinline__ + void take_action_when_have_nan(const TopkSelectArgs &args, uint32_t batch_idx) { + if (args.abort_when_nan_found) { + if (threadIdx.x == 0) { + printf("[topk_select] NaN detected. Calling `trap;` which will result in \"unspecified launch failure\"\n"); + } + __syncthreads(); + ku::trap(); + } else { + if (threadIdx.x == 0) { + *((OutIdxT*)args.output_index + batch_idx * args.stride_output_index_batch) = 0x3F3F3F3F; + } + } + } +}; + +template +class TopkSelectKernelBF16Base : public TopkSelectKernelBase { + using Base = TopkSelectKernelBase; +public: + using ValueT = typename Base::ValueT; + using OutIdxT = typename Base::OutIdxT; + using TmaParams = typename Base::TmaParams; + using SharedMemoryPlanBase = typename Base::SharedMemoryPlanBase; + using EpilogueT = typename Base::EpilogueT; + using Base::NUM_THREADS; + using Base::NUM_WARPS; + using Base::MAX_TOPK; + using Base::NUM_ELEMS_PER_128b; + using Base::NUM_UINT32_PER_128b; + using Base::NUM_ELEMS_PER_ROUND; + using Base::NUM_ELEMS_PER_THREAD_PER_ROUND; + using Base::ELEMS_PER_THREAD_PER_ROUND_MASK; + using Base::NUM_128b_PER_THREAD_PER_ROUND; + using Base::NUM_ELEMS_PER_SEG; + using Base::NUM_TAIL_ELEMS; + using Base::NUM_TAIL_SEGS; + using Base::NUM_ELEMS_IN_INIT_WINDOW; + using Base::NUM_SEGS_IN_INIT_WINDOW; + using Base::NUM_INIT_ROUNDS_MAX; + using Base::NUM_UINT32_IN_INIT_WINDOW_PER_THREAD; + using Base::NUM_128b_INIT_PER_THREAD; + using Base::NUM_SEGS_PER_ROUND; + using Base::NUM_SEGS_PER_ISSUE_WARP; + using Base::NUM_ISSUE_WARPS; + using Base::NUM_TMA_LOAD_BUFS; + using Base::TMA_PREFETCH_DEPTH; + using Base::RECONSTRUCT_THRESHOLD; + using Base::PERM_ADD_BASE; + using Base::PERM_MUL_PRIME; + using Base::NUM_EXTRA_SLOTS; + using Base::PLACEHOLDER_PAIR; + using Base::NUM_RECONSTRUCT_BUCKETS; + using Base::NUM_RECONSTRUCT_UNITS_MAX; + + static_assert(std::is_same_v); + static_assert(!Config::sorted_value); + static constexpr uint32_t NUM_UINT32_RECONSTRUCT_PER_THREAD = + ((NUM_RECONSTRUCT_UNITS_MAX + NUM_THREADS - 1) / NUM_THREADS) | 1u; // padding + static_assert(2 * NUM_UINT32_RECONSTRUCT_PER_THREAD <= 256); + static constexpr uint16_t PLACEHOLDER = 0xff80; // -INF + static constexpr uint32_t NEG_INF_X2_BITS = 0xFF80FF80u; + + // Histogram pass 1 (MSB radix): for each distorted bf16x2 value, retrieve the upper 8 bits and performs atomicAdd on the bucket counter + // `num_packed_values` should be guaranteed to be aligned by `NUM_PACKED_VALUES_ALIGNMENT` + template + static __device__ __forceinline__ + void histogram_radix_msb(uint32_t *bucket_counter, const nv_bfloat162 (&packed_values)[N], uint32_t num_packed_values) { + uint32_t bucket_base = cute::cast_smem_ptr_to_uint(bucket_counter); + CUTE_UNROLL + for (uint32_t i = 0; i < N; i++) { + if (i % NUM_PACKED_VALUES_ALIGNMENT == 0 && i == num_packed_values) break; + uint32_t raw = bf16x2_to_u32(packed_values[i]); + uint32_t distorted; + distort_x2((uint16_t*)&distorted, (const uint16_t*)&raw); + // bfe + mad form each bucket address in two instructions, then two shared-memory adds. + asm volatile ( + "{\n" + ".reg .b32 i0, i1, a0, a1;\n" + "bfe.u32 i0, %0, 8, 8;\n" + "bfe.u32 i1, %0, 24, 8;\n" + "mad.lo.u32 a0, i0, 4, %1;\n" + "mad.lo.u32 a1, i1, 4, %1;\n" + "red.shared.add.u32 [a0], 1;\n" + "red.shared.add.u32 [a1], 1;\n" + "}\n" + : + : "r"(distorted), "r"(bucket_base) + : "memory" + ); + } + } + + // Histogram pass 2 (LSB radix refinement): among values whose distorted MSB equals pivot_hi8, retrieve the lower 8 bits and performs atomicAdd on the bucket counter + // `num_packed_values` should be guaranteed to be aligned by `NUM_PACKED_VALUES_ALIGNMENT` + // `pivot_hi8` is the highest 8 bit of the DISTORTED pivot + template + static __device__ __forceinline__ + void histogram_radix_lsb_for_pivot_msb(uint32_t *bucket_counter, uint32_t pivot_hi8, const nv_bfloat162 (&packed_values)[N], uint32_t num_packed_values) { + uint32_t bucket1_base = cute::cast_smem_ptr_to_uint(bucket_counter); + // Every elements in the pivot bucket must have the same sign, so we can use this information to optimize instead of + bool bucket_negative = pivot_hi8 < 0x80; + uint32_t orig_pivot_hi8 = bucket_negative ? 0xFF - pivot_hi8 : pivot_hi8 - 0x80; // Un-distort + uint32_t raw_hi8_x2 = (orig_pivot_hi8 << 16) | orig_pivot_hi8; + uint32_t low_byte_xor = bucket_negative ? 0xFFFFFFFFu : 0u; + // Elements outside the bucket are counted in this row's sink slot (index NUM_RECONSTRUCT_BUCKETS) and never read. + uint32_t scratch_bucket_idx_x2 = 0x01000100; // 2x 256 + CUTE_UNROLL + for (uint32_t i = 0; i < N; i++) { + if (i % NUM_PACKED_VALUES_ALIGNMENT == 0 && i == num_packed_values) break; + asm volatile ( + "{\n" + ".reg .b32 hi, lo, sel, t, t0, t1, a0, a1;\n" + "prmt.b32 hi, %0, 0, 0x5341;\n" // bytes {1, zero, 3, zero}: both raw high bytes as 0x00hh halves + "set.eq.s32.bf16x2 sel, hi, %1;\n" // 0xFFFF per half whose raw high byte equals the pivot's + "lop3.b32 lo, %0, %4, 0x00ff00ff, 0x28;\n" // lo = (%0 ^ %4) & 0x00ff00ff + "lop3.b32 t, sel, lo, %3, 0xca;\n" // t = sel ? lo : %3, per-bit + "and.b32 t0, t, 0xFFFF;\n" + "shr.u32 t1, t, 16;\n" + "mad.lo.u32 a0, t0, 4, %2;\n" // base + bucket * 4 + "mad.lo.u32 a1, t1, 4, %2;\n" + "red.shared.add.u32 [a0], 1;\n" + "red.shared.add.u32 [a1], 1;\n" + "}\n" + : + : "r"(bf16x2_to_u32(packed_values[i])), "r"(raw_hi8_x2), "r"(bucket1_base), "r"(scratch_bucket_idx_x2), "r"(low_byte_xor) + : "memory" + ); + } + } + + // Count the number of elements that are 1) > pivot 2) = pivot 3) NaN in the given array + struct CensusCounts { uint32_t cnt_gt; uint32_t cnt_eq; uint32_t cnt_nan; }; + // `num_packed_values` should be guaranteed to be aligned by `NUM_PACKED_VALUES_ALIGNMENT` + template + static __device__ __forceinline__ + CensusCounts get_census_counts(const nv_bfloat162 (&values)[N], uint32_t num_packed_values, uint32_t pivot_value_x2_bits) { + static_assert(2 * N <= 256); // Since we're going to use bf16 for accumulation + nv_bfloat162 gt_accum = {0.0f, 0.0f}; + nv_bfloat162 eq_accum = {0.0f, 0.0f}; + nv_bfloat162 nan_accum = {0.0f, 0.0f}; + CUTE_UNROLL + for (uint32_t i = 0; i < N; i++) { + if (i % NUM_PACKED_VALUES_ALIGNMENT == 0 && i == num_packed_values) break; + uint32_t raw = bf16x2_to_u32(values[i]); + asm volatile ( + "{\n" + ".reg .b32 gt_result, eq_result;\n" + "set.gt.bf16x2.bf16x2 gt_result, %3, %4;\n" + "add.rn.bf16x2 %0, gt_result, %0;\n" + "set.eq.bf16x2.bf16x2 eq_result, %3, %4;\n" + "add.rn.bf16x2 %1, eq_result, %1;\n" + "min.NaN.bf16x2 %2, %2, %3;\n" // We use `min.NaN` for NaN detection + "}\n" + : "+r"(*(uint32_t*)>_accum), "+r"(*(uint32_t*)&eq_accum), "+r"(*(uint32_t*)&nan_accum) + : "r"(raw), "r"(pivot_value_x2_bits) + ); + } + uint32_t cnt_gt = (uint32_t)(float)(gt_accum.x + gt_accum.y); + uint32_t cnt_eq = (uint32_t)(float)(eq_accum.x + eq_accum.y); + uint32_t nan_flag; + asm ("set.nan.bf16x2.bf16x2 %0, %1, %1;" : "=r"(nan_flag) : "r"(*(uint32_t*)&nan_accum)); + uint32_t cnt_nan = nan_flag != 0 ? 1u : 0u; + return {cnt_gt, cnt_eq, cnt_nan}; + } + + // Decide whether or not to accept an element in the old survivor buffer + // If taken, update relevant status (`dst_ptr` and `eq_quota`) + template + static __device__ __forceinline__ + void copy_selected_pairs_to_survivor( + uint32_t &dst_ptr, // The pointer to the new survivor buffer. May increase. Marked as uint32 since it's obtained from cast_smem_ptr_to_uint + uint32_t &eq_quota, // The EQ quota of this thread. May decrease + nv_bfloat162 packed_values, + uint32_t pivot_value_x2_bits, + uint32_t src_ptr // The pointer to the location of src. `src` contains 8B pairs in {index, value} + ) { + uint32_t values_raw = bf16x2_to_u32(packed_values); + #define MOVE_SELECTED_PAIR2(ADD) \ + asm volatile ( \ + "{\n" \ + ".reg .pred g0, g1, e0, e1, q, t0, t1, s0, s1;\n" \ + ".reg .b32 pair0_lo, pair0_hi, pair1_lo, pair1_hi;\n" \ + "ld.shared.v4.b32 {pair0_lo, pair0_hi, pair1_lo, pair1_hi}, [%4];\n" \ + "setp.gt.bf16x2 g0|g1, %2, %3;\n" \ + "setp.eq.bf16x2 e0|e1, %2, %3;\n" \ + /* element 0 (low half) */ \ + "setp.ne.and.u32 t0, %1, 0, e0;\n" \ + "@t0 sub.u32 %1, %1, 1;\n" \ + "or.pred s0, g0, t0;\n" \ + "@s0 st.shared.v2.b32 [%0], {pair0_lo, pair0_hi};\n" \ + "@s0 " ADD "\n" \ + /* element 1 (high half; quota is re-tested after element 0's decrement) */ \ + "setp.ne.and.u32 t1, %1, 0, e1;\n" \ + "@t1 sub.u32 %1, %1, 1;\n" \ + "or.pred s1, g1, t1;\n" \ + "@s1 st.shared.v2.b32 [%0], {pair1_lo, pair1_hi};\n" \ + "@s1 " ADD "\n" \ + "}\n" \ + : "+r"(dst_ptr), "+r"(eq_quota) \ + : "r"(values_raw), "r"(pivot_value_x2_bits), "r"(src_ptr) \ + : "memory" \ + ) + if constexpr (USE_CLUSTER_ADDRESSING) { + MOVE_SELECTED_PAIR2("add.u32 %0, %0, 8;"); + } else { + MOVE_SELECTED_PAIR2("add.f32 %0, %0, 0f00000008;"); + } + #undef MOVE_SELECTED_PAIR2 + } + + // The src-in-register version of `copy_selected_pairs_to_survivor` + template + static __device__ __forceinline__ + void append_selected_pairs_from_registers( + uint32_t &dst_ptr, + uint32_t &eq_quota, + nv_bfloat162 packed_values, + uint32_t pivot_value_x2_bits, + uint32_t index0, + uint32_t index1 + ) { + uint32_t values_raw = bf16x2_to_u32(packed_values); + uint32_t val_word1 = values_raw >> 16; // element 1 low, clean high + #define APPEND_SELECTED_PAIR2(ADD) \ + asm volatile ( \ + "{\n" \ + ".reg .pred g0, g1, e0, e1, q, t0, t1, s0, s1;\n" \ + "setp.gt.bf16x2 g0|g1, %2, %3;\n" \ + "setp.eq.bf16x2 e0|e1, %2, %3;\n" \ + /* element 0 (low half) */ \ + "setp.ne.and.u32 t0, %1, 0, e0;\n" \ + "@t0 sub.u32 %1, %1, 1;\n" \ + "or.pred s0, g0, t0;\n" \ + "@s0 st.shared.v2.u32 [%0], {%4, %2};\n" \ + "@s0 " ADD "\n" \ + /* element 1 (high half; quota is re-tested after element 0's decrement) */ \ + "setp.ne.and.u32 t1, %1, 0, e1;\n" \ + "@t1 sub.u32 %1, %1, 1;\n" \ + "or.pred s1, g1, t1;\n" \ + "@s1 st.shared.v2.u32 [%0], {%5, %6};\n" \ + "@s1 " ADD "\n" \ + "}\n" \ + : "+r"(dst_ptr), "+r"(eq_quota) \ + : "r"(values_raw), "r"(pivot_value_x2_bits), \ + "r"(index0), "r"(index1), "r"(val_word1) \ + : "memory" \ + ) + if constexpr (USE_CLUSTER_ADDRESSING) { + APPEND_SELECTED_PAIR2("add.u32 %0, %0, 8;"); + } else { + APPEND_SELECTED_PAIR2("add.f32 %0, %0, 0f00000008;"); + } + #undef APPEND_SELECTED_PAIR2 + } + + // Get the pivot via a two-stage histogram and get PivotAndQuota + // The MSB histogram should be ready and all threads should be able to see it (e.g. via a `__syncthreads()`) before we call the function + // We don't absorb MSB histogram's logic inside since sometimes we'd like to overlap TMA load and MSB histogram's modification + struct PivotAndQuota { + uint32_t pivot_value_x2_bits; + uint32_t start_pos_in_collector; + uint32_t eq_quota; + uint32_t cnt_nan; + }; + // `num_my` should be guaranteed to be aligned by `NUM_PACKED_VALUES_ALIGNMENT` + template + static __device__ __forceinline__ + PivotAndQuota compute_pivot_and_quota(uint32_t topk, const nv_bfloat162 (&values)[N], + uint32_t num_values, uint32_t warp_idx, uint32_t lane_idx, SharedMemoryPlanBase &smem) { + if (warp_idx == 0) { + Base::template find_pivot_in_histogram(smem, smem.reconstruct_bucket_counter[0], topk, lane_idx); + } + __syncthreads(); + uint32_t pivot_hi8 = smem.reconstruct_pivot_bucket; + uint32_t num_elem_should_select_in_pivot_bucket = smem.reconstruct_num_should_select; + + histogram_radix_lsb_for_pivot_msb(smem.reconstruct_bucket_counter[1], pivot_hi8, values, num_values); + __syncthreads(); + + if (warp_idx == 0) { + Base::template find_pivot_in_histogram(smem, smem.reconstruct_bucket_counter[1], num_elem_should_select_in_pivot_bucket, lane_idx); + } + __syncthreads(); + uint32_t pivot_lo8 = smem.reconstruct_pivot_bucket; + uint32_t pivot_distorted = (pivot_hi8 << 8) | pivot_lo8; + uint16_t pivot_value = un_distort((uint16_t)pivot_distorted); + uint32_t pivot_value_x2_bits = ((uint32_t)pivot_value << 16) | pivot_value; + + auto census = get_census_counts(values, num_values, pivot_value_x2_bits); + + static_assert(NUM_WARPS <= NUM_RECONSTRUCT_BUCKETS); + auto eqgt = Base::compute_equal_quota_and_prefix(census.cnt_gt, census.cnt_eq, topk, warp_idx, lane_idx, smem.reconstruct_bucket_counter[0]); + return {pivot_value_x2_bits, eqgt.start_pos_in_collector, eqgt.eq_quota, census.cnt_nan}; + } + + // Main part of the normal kernel and phase A of the cluster kernel. + // The input is divided into: + // - a "perm" prefix [0, num_perm_segs * NUM_ELEMS_PER_SEG), visited in pseudo-random order; + // - a "tail" suffix [num_perm_segs * NUM_ELEMS_PER_SEG, end_vocab_idx), visited sequentially. + // The init phase loads the tail plus the first perm segments into one init window, selects + // the initial top-k from it, then the main loop continues over the remaining perm segments. + // + // Returns the number of real selected pairs in the final survivor buffer; this is normally + // `topk`, but can be smaller in a cluster CTA whose local range has fewer real elements. + template + static __device__ __forceinline__ + uint32_t scan_segs(const TmaParams &tma_params, SharedMemoryPlanBase &smem, + uint32_t batch_idx, uint32_t end_vocab_idx, uint32_t topk, + uint32_t warp_idx, uint32_t lane_idx, + uint32_t num_perm_segs, // The number of segments to be permuted, globally + uint32_t local_start_seg_idx, // The index of the first "local" segment ("local" means "belong to this CTA"). 0 if not the cluster-based implementation + uint32_t num_local_perm_segs, // The number of permuted segments to be processed, locally + uint32_t num_local_tail_elems_padded, // num_local_tail_elems, padded to NUM_TAIL_ELEMS if num_perm_segs > 0, otherwise padded to NUM_ELEMS_PER_SEG + uint32_t num_local_tail_elems, // The number of tail elems locally + uint32_t &survivor_buf_idx, + bool &have_nan, + IsWarpActiveF &&is_warp_active_f) { + uint32_t num_local_elems_padded = num_local_tail_elems_padded + num_local_perm_segs * NUM_ELEMS_PER_SEG; + uint32_t num_local_rounds = ku::ceil_div(num_local_elems_padded, (uint32_t)NUM_ELEMS_PER_ROUND); + uint32_t num_init_rounds = min(num_local_rounds, (uint32_t)NUM_INIT_ROUNDS_MAX); + uint32_t num_local_tail_segs_padded = num_local_tail_elems_padded / NUM_ELEMS_PER_SEG; + uint32_t num_main_rounds = num_local_rounds - num_init_rounds; + + // Permutation generation arguments + uint32_t perm_len = max(num_perm_segs, 1u); + uint32_t perm_mul = PERM_MUL_PRIME % perm_len; + + // Permutation state + // Only valid for the lane elected by `elect_one_sync` + uint32_t tma_permuted_segment_stride = NUM_ISSUE_WARPS * perm_mul % perm_len; + uint32_t linear_tma_segment_start = local_start_seg_idx + warp_idx; + uint32_t next_tma_permuted_segment = Base::get_permuted_seg_idx(linear_tma_segment_start, perm_len, perm_mul); + + auto issue_tma_copy = [&](uint32_t round_idx) { + Base::template issue_tma_loads_for_round( + smem, tma_params.tensor_map, + batch_idx, end_vocab_idx, + num_perm_segs, round_idx, + next_tma_permuted_segment, tma_permuted_segment_stride, + warp_idx + ); + }; + auto issue_tma_copy_for_main_rounds = [&](uint32_t round_idx) { + issue_tma_copy.template operator()(round_idx); + }; + + uint32_t threshold_x2_bits = NEG_INF_X2_BITS; + uint32_t num_incomers = 0; + uint32_t num_survivors = 0; + + // Init phase + if (num_local_elems_padded != 0) { + // Issue copy: Copy the tail and the first few segments, result in init_buf[0: NUM_ELEMS_IN_INIT_WINDOW] (or init_buf[0: ]) + ValueT *init_buf = (ValueT*)smem.incoming_topk_pairs; + if (cute::elect_one_sync()) { + if (num_local_tail_elems_padded != 0) { + issue_tma_copy.template operator()(0); + } else { + issue_tma_copy.template operator()(0); + } + + CUTE_UNROLL + for (uint32_t i = 1; i < NUM_INIT_ROUNDS_MAX; i++) { + if (i < num_init_rounds) { + issue_tma_copy.template operator()(i); + } + } + + // Pre-issue TMA copies for the main part + CUTE_UNROLL + for (uint32_t i = 0; i < TMA_PREFETCH_DEPTH; i++) { + if (i < num_main_rounds) { + issue_tma_copy_for_main_rounds(i); + } + } + } + + if (num_perm_segs != 0 && num_local_tail_elems_padded != 0) { + Base::fill_padded_tail_segments(init_buf, num_local_tail_elems); + __syncthreads(); + } + + // Decide how many 128-bit elements each thread processes. + uint32_t num_elems_in_init_window_padded = num_local_tail_elems_padded + min(num_local_perm_segs, (uint32_t)NUM_SEGS_IN_INIT_WINDOW - num_local_tail_segs_padded) * NUM_ELEMS_PER_SEG; // Init window size, padded to NUM_ELEMS_PER_SEG + uint32_t num_128b_in_init_window_padded = num_elems_in_init_window_padded / NUM_ELEMS_PER_128b; + uint32_t cnt_floor = num_128b_in_init_window_padded / NUM_THREADS; + uint32_t cnt_rem = num_128b_in_init_window_padded % NUM_THREADS; + uint32_t my_elem_start_idx = (threadIdx.x * cnt_floor + min(threadIdx.x, cnt_rem)) * NUM_ELEMS_PER_128b; + uint32_t num_my_elems = NUM_ELEMS_PER_128b * cnt_floor + (threadIdx.x < cnt_rem ? NUM_ELEMS_PER_128b : 0u); + uint32_t tail_padding_elems = num_local_tail_elems_padded - num_local_tail_elems; + uint32_t num_real_init_elems = num_elems_in_init_window_padded - tail_padding_elems; + num_survivors = min(topk, num_real_init_elems); + + CUTE_UNROLL + for (uint32_t i = 0; i < NUM_INIT_ROUNDS_MAX; i++) { + if (i == num_init_rounds) break; + smem.init_full_bar[i].wait(0); + + // Fill the incomplete segment with PLACEHOLDER (-INF) + if (num_local_tail_elems % NUM_ELEMS_PER_SEG != 0 && i == num_local_tail_elems / NUM_ELEMS_PER_ROUND) { + uint32_t box_end = ku::ceil_div(num_local_tail_elems, (uint32_t)NUM_ELEMS_PER_SEG) * NUM_ELEMS_PER_SEG; + for (uint32_t e = num_local_tail_elems + threadIdx.x; e < box_end; e += NUM_THREADS) { + init_buf[Base::sw_elem(e)] = __ushort_as_bfloat16(PLACEHOLDER); + } + __syncthreads(); + } + nv_bfloat162 my_values[4 * NUM_128b_PER_THREAD_PER_ROUND]; + uint32_t num_my_values = 0; + CUTE_UNROLL + for (uint32_t k = 0; k < NUM_128b_PER_THREAD_PER_ROUND; ++k) { + uint32_t pos = threadIdx.x + (i * NUM_128b_PER_THREAD_PER_ROUND + k) * NUM_THREADS; + if (pos < num_128b_in_init_window_padded) { + ld_shared<4>(my_values + k * 4, reinterpret_cast(init_buf + Base::sw_b128(pos) * NUM_ELEMS_PER_128b)); + num_my_values = 4 * (k + 1); + } + } + histogram_radix_msb<4 * NUM_128b_PER_THREAD_PER_ROUND, 4>(smem.reconstruct_bucket_counter[0], my_values, num_my_values); + } + + nv_bfloat162 init_values[NUM_UINT32_IN_INIT_WINDOW_PER_THREAD]; + CUTE_UNROLL + for (uint32_t j = 0; j < NUM_128b_INIT_PER_THREAD; ++j) { + if (j * NUM_ELEMS_PER_128b == num_my_elems) break; + uint32_t u0 = my_elem_start_idx / NUM_ELEMS_PER_128b + j; + ld_shared<4>(init_values + j * NUM_UINT32_PER_128b, reinterpret_cast(init_buf + Base::sw_b128(u0) * NUM_ELEMS_PER_128b)); + } + + __syncthreads(); // publish the round-1 histogram + + static_assert(NUM_ELEMS_IN_INIT_WINDOW <= 0xFFFF); + // The CTA's init-window slice may contain fewer real elements than K (e.g. a cluster + // rank whose visit range mostly overlaps the padded tail). In that case the pivot + // selection must degrade to "select all real elements": cap the K used in the pivot / + // quota computation at the real count + uint32_t effective_topk = min(topk, num_real_init_elems); + auto [pivot_value_x2_bits, start_pos_in_collector, eq_quota, cnt_nan] = + compute_pivot_and_quota(effective_topk, init_values, num_my_elems / 2, warp_idx, lane_idx, smem); + // The init census walks the thread's whole slice of the window, i.e. every element of the init + // window exactly once, so NaNs inside the window are caught here. + have_nan |= cnt_nan != 0; + + { + uint32_t smem_base = cute::cast_smem_ptr_to_uint(smem.surviving_topk_pairs[0]); + uint32_t out_ptr = smem_base + start_pos_in_collector * (uint32_t)sizeof(uint64_t); + uint32_t s0 = local_start_seg_idx + (max(my_elem_start_idx, num_local_tail_elems_padded) - num_local_tail_elems_padded) / NUM_ELEMS_PER_SEG; + uint32_t perm_state = Base::get_permuted_seg_idx(s0, perm_len, perm_mul); + float unit_base_f = 0.0f; + CUTE_UNROLL + for (uint32_t i = 0; i < NUM_UINT32_IN_INIT_WINDOW_PER_THREAD; i++) { + // num_my_elems is a multiple of NUM_ELEMS_PER_128b (= 8), so i*2 == num_my_elems + // only at i % 4 == 0; the extra condition lets the compiler unroll by 4. + if (i % 4 == 0 && i*2 == num_my_elems) break; + if (i % 4 == 0) { + uint32_t g_u = my_elem_start_idx + i / 4 * NUM_ELEMS_PER_128b; + if (g_u < num_local_tail_elems_padded) { + unit_base_f = __uint_as_float(num_perm_segs * NUM_ELEMS_PER_SEG + g_u); + } else { + unit_base_f = __uint_as_float(perm_state * NUM_ELEMS_PER_SEG + g_u % NUM_ELEMS_PER_SEG); + if (g_u % NUM_ELEMS_PER_SEG == NUM_ELEMS_PER_SEG - NUM_ELEMS_PER_128b) { + Base::advance_perm_state(perm_state, perm_mul, perm_len); + } + } + } + // fp32-subnormal FP-pipe int-add: index = base + element offset + uint32_t index0 = __float_as_uint(unit_base_f + __uint_as_float(i % 4 * 2)); + uint32_t index1 = __float_as_uint(unit_base_f + __uint_as_float(i % 4 * 2 + 1)); + nv_bfloat162 cur_value = init_values[i]; // two bf16 payloads packed as bf16x2 + append_selected_pairs_from_registers(out_ptr, eq_quota, cur_value, pivot_value_x2_bits, index0, index1); + } + } + + threshold_x2_bits = pivot_value_x2_bits; + } + __syncthreads(); + + + uint32_t linear_segment_start = + local_start_seg_idx + + num_init_rounds * NUM_SEGS_PER_ROUND + - num_local_tail_segs_padded + + warp_idx; + uint32_t current_permuted_segment = Base::get_permuted_seg_idx(linear_segment_start, perm_len, perm_mul); + uint32_t permuted_segment_stride_per_round = NUM_SEGS_PER_ROUND % perm_len * perm_mul % perm_len; + + // Re-select the top-k elements from the smem.surviving_topk_pairs[survivor_buf_idx][:num_survivors] + smem.incoming_topk_pairs[:num_incomers], + // Write the selected pairs into the other survivor buffer, and return the new pivot + auto reconstruct = [&](uint32_t cur_extra_len) -> uint32_t { + uint32_t tid = threadIdx.x; + Base::clear_reconstruct_histograms(smem, tid); + + uint32_t padded_extra_len = ku::ceil(cur_extra_len, 2u); // Since we process elements in pairs (uint32_t) + if (warp_idx == 0 && (cur_extra_len & 1u)) { + smem.incoming_topk_pairs[cur_extra_len] = PLACEHOLDER_PAIR; + } + __syncthreads(); + + // units [0, MAX_TOPK/2): the current survivor buffer + // The rest: incomers + uint32_t num_uint32 = (MAX_TOPK + padded_extra_len) / 2; + uint32_t num_uint32_per_thread = ku::ceil_div(num_uint32, (uint32_t)NUM_THREADS) | 1u; // force odd => bank-conflict-free smem gathers + uint32_t unit_base = tid * num_uint32_per_thread; + uint32_t num_my_units = unit_base < num_uint32 ? min(num_uint32_per_thread, num_uint32 - unit_base) : 0u; + + auto unit_to_pair_addr = [&](uint32_t offset) -> uint32_t { + return offset < MAX_TOPK / 2 + ? cute::cast_smem_ptr_to_uint(smem.surviving_topk_pairs[survivor_buf_idx] + 2 * offset) + : cute::cast_smem_ptr_to_uint(smem.incoming_topk_pairs + (2 * offset - MAX_TOPK)); + }; + + nv_bfloat162 values[NUM_UINT32_RECONSTRUCT_PER_THREAD]; + CUTE_UNROLL + for (uint32_t m = 0; m < NUM_UINT32_RECONSTRUCT_PER_THREAD; m++) { + if (m == num_my_units) break; + uint32_t pair2[4]; + ld_shared<4>(pair2, (const uint32_t*)unit_to_pair_addr(unit_base + m)); + // One b128 unit = 2 pairs {index0, value0, index1, value1}; pick the low halves of the + // two value words (the bf16 payloads) and pack them into one bf16x2 word. + values[m] = u32_to_bf16x2(__byte_perm(pair2[1], pair2[3], 0x5410)); + } + + histogram_radix_msb(smem.reconstruct_bucket_counter[0], values, num_my_units); + __syncthreads(); + + auto [pivot_value_x2_bits, out_prefix, eq_quota, cnt_nan] = + compute_pivot_and_quota(topk, values, num_my_units, warp_idx, lane_idx, smem); + // Every NaN that the main loop collected is part of the buffer this census just walked (the hit + // test is `.gtu`, so NaN always becomes an incomer). + have_nan |= cnt_nan != 0; + + { // write back + uint32_t out_ptr = cute::cast_smem_ptr_to_uint(smem.surviving_topk_pairs[survivor_buf_idx ^ 1]) + out_prefix * (uint32_t)sizeof(uint64_t); + CUTE_UNROLL + for (uint32_t m = 0; m < NUM_UINT32_RECONSTRUCT_PER_THREAD; m++) { + if (m == num_my_units) break; + copy_selected_pairs_to_survivor(out_ptr, eq_quota, values[m], pivot_value_x2_bits, unit_to_pair_addr(unit_base + m)); + } + } + + survivor_buf_idx ^= 1; + return pivot_value_x2_bits; + }; + + uint32_t logical_elem_offset = threadIdx.x * NUM_ELEMS_PER_THREAD_PER_ROUND; + uint32_t offset_in_segment = logical_elem_offset % NUM_ELEMS_PER_SEG; + + uint32_t swizzle_mask = Base::sw_msk(logical_elem_offset); + uint32_t chunk_swizzle_mask = swizzle_mask & ELEMS_PER_THREAD_PER_ROUND_MASK; + uint32_t smem_read_offset = logical_elem_offset ^ (swizzle_mask & ~ELEMS_PER_THREAD_PER_ROUND_MASK); + + // Track the TMA buffer index and its phase incrementally instead of dividing every round. + // On long rows the init window is a tiny fraction of the row, so its pivot stays far above the + // row's true one and the append rate (hence the store path) stays high for the whole scan: + // reconstruct early to tighten the pivot. Short rows would pay more for the reconstruct than the + // extra appends cost, so they keep the plain threshold. + uint32_t reconstruct_trigger = num_main_rounds > 16 ? RECONSTRUCT_THRESHOLD / 4 : RECONSTRUCT_THRESHOLD; + + uint32_t tma_buf_idx = 0; + uint32_t tma_buf_phase = 0; + + for (uint32_t main_round_idx = 0; main_round_idx < num_main_rounds; ++main_round_idx) { + if (main_round_idx + TMA_PREFETCH_DEPTH < num_main_rounds && cute::elect_one_sync()) { + issue_tma_copy_for_main_rounds(main_round_idx + TMA_PREFETCH_DEPTH); + } + + bool is_warp_active = is_warp_active_f(main_round_idx); + + const ValueT *buf = smem.tma_load_buf[tma_buf_idx]; + // One bit per element of this thread's slice, in element order. + // Hits are rare, so appending through the set bits of this mask is faster than testing + storing every element again. + static_assert(NUM_ELEMS_PER_THREAD_PER_ROUND <= 32); // hit_mask is a uint32_t + static_assert(NUM_ELEMS_PER_THREAD_PER_ROUND % 4 == 0); // the loop below packs 4 elements per nibble + uint32_t hit_mask = 0; + if (is_warp_active) { + smem.tma_load_full_bar[tma_buf_idx].wait(tma_buf_phase); + ValueT values[NUM_ELEMS_PER_THREAD_PER_ROUND]; + Base::template load_swizzled_slice(values, buf, smem_read_offset, chunk_swizzle_mask); + CUTE_UNROLL + for (int32_t j = NUM_ELEMS_PER_THREAD_PER_ROUND - 4; j >= 0; j -= 4) { + asm volatile ( + "{\n" + ".reg .b32 r0, r1, ind, nib;\n" + "set.gtu.s32.bf16x2 r0, %1, %3;\n" // .gtu: NaN always counts as a hit, so every NaN reaches the incoming buffer and the census below reports it. + "set.gtu.s32.bf16x2 r1, %2, %3;\n" // `set` with .s32 fills the corresponding 16bit to 0xFFFF (-1) when the condition holds. We use `prmt` to extract one bit from each + "prmt.b32 ind, r0, r1, 0x7531;\n" + "dp4a.s32.s32 nib, ind, 0xF8FCFEFF, 0;\n" // ind's bytes are 0x00/0xFF, so with signed byte weights each hit contributes 1/2/4/8 + "mad.lo.u32 %0, %0, 16, nib;\n" + "}\n" + : "+r"(hit_mask) + : "r"(*(const uint32_t*)(values + j)), + "r"(*(const uint32_t*)(values + j + 2)), + "r"(threshold_x2_bits) + ); + } + } + uint32_t num_new_incomers = __popc(hit_mask); + + uint32_t warp_total_hits = __reduce_add_sync(0xFFFFFFFF, num_new_incomers); + if (lane_idx == 0) { + smem.warp_cnt[warp_idx] = warp_total_hits; + } + __syncthreads(); + + static_assert(NUM_WARPS <= 32); + uint32_t stored_warp_hits = lane_idx < NUM_WARPS ? smem.warp_cnt[lane_idx] : 0u; + uint32_t num_total_hits_in_this_round = __reduce_add_sync(0xFFFFFFFF, stored_warp_hits); + + uint32_t seg_elem_base = current_permuted_segment * NUM_ELEMS_PER_SEG + offset_in_segment; + + // Element e of this thread's slice lives at smem_read_offset + (e ^ chunk_swizzle_mask) + uint32_t elem_addr_base = cute::cast_smem_ptr_to_uint(buf + smem_read_offset); + + // Start the first hit's smem load before the prefix scan / count exchange below, so that its + // latency (and the barrier wait) overlaps with them instead of delaying the first store. + uint32_t first_hit_e = hit_mask != 0 ? __ffs(hit_mask) - 1u : 0u; + uint32_t first_hit_val = 0; + if (is_warp_active && warp_total_hits != 0) { + asm volatile ("ld.shared.u16 %0, [%1];" : "=r"(first_hit_val) : "r"(elem_addr_base + ((first_hit_e ^ chunk_swizzle_mask) << 1))); + } + + // Lane prefix via one ballot per bit of the (<= 16) hit count: the ballots are independent, + // so this is much shorter on the critical path than a 5-step shuffle scan. + static_assert(NUM_ELEMS_PER_THREAD_PER_ROUND < 32); // one ballot per bit of num_new_incomers + uint32_t lane_prefix = 0; + CUTE_UNROLL + for (uint32_t k = 0; k < 5; ++k) { + uint32_t bit = __ballot_sync(0xFFFFFFFF, (num_new_incomers >> k) & 1u) & ((1u << lane_idx) - 1u); + lane_prefix += (uint32_t)__popc(bit) << k; + } + + uint32_t dst_slot = + num_incomers + + __reduce_add_sync(0xFFFFFFFF, lane_idx < warp_idx ? stored_warp_hits : 0u) + + lane_prefix; + + if (is_warp_active && warp_total_hits != 0) { + uint32_t dst_ptr = cute::cast_smem_ptr_to_uint(smem.incoming_topk_pairs) + dst_slot * (uint32_t)sizeof(uint64_t); + if (hit_mask != 0) { + asm volatile ("st.shared.v2.u32 [%0], {%1, %2};" :: "r"(dst_ptr), "r"(seg_elem_base + first_hit_e), "r"(first_hit_val) : "memory"); + dst_ptr += (uint32_t)sizeof(uint64_t); + } + uint32_t mask = hit_mask & (hit_mask - 1u); + while (mask != 0) { + uint32_t e = __ffs(mask) - 1u; + mask &= mask - 1u; + uint32_t val_word; + asm volatile ("ld.shared.u16 %0, [%1];" : "=r"(val_word) : "r"(elem_addr_base + ((e ^ chunk_swizzle_mask) << 1))); + asm volatile ("st.shared.v2.u32 [%0], {%1, %2};" :: "r"(dst_ptr), "r"(seg_elem_base + e), "r"(val_word) : "memory"); + dst_ptr += (uint32_t)sizeof(uint64_t); + } + } + Base::advance_perm_state(current_permuted_segment, permuted_segment_stride_per_round, perm_len); + + num_incomers += num_total_hits_in_this_round; + // No warp may run ahead into the next round: the TMA copy issued there reuses the smem + // buffer this round is reading, and warp_cnt must not be overwritten while others read it. + __syncthreads(); + + if (++tma_buf_idx == NUM_TMA_LOAD_BUFS) { + tma_buf_idx = 0; + tma_buf_phase ^= 1u; + } + + // On long rows the init window is a tiny fraction of the row, so its pivot stays far above the + // row's true one and the append rate (hence the store path) stays high for the whole scan: + // reconstruct early to tighten the pivot. Short rows would pay more for the reconstruct than + // the extra appends cost, so they keep the plain threshold. + if (num_incomers >= reconstruct_trigger && main_round_idx + 1 < num_main_rounds) { + threshold_x2_bits = reconstruct(num_incomers); + num_incomers = 0; + } + } + + if (num_incomers > 0) { + reconstruct(num_incomers); + num_incomers = 0; + } + + return num_survivors; + } +}; + +} // namespace topk_select_common diff --git a/python/sglang/kernels/jit/csrc/deepselect/vendor/cuda_kernels/config.h b/python/sglang/kernels/jit/csrc/deepselect/vendor/cuda_kernels/config.h new file mode 100644 index 0000000000..2c1d4fa151 --- /dev/null +++ b/python/sglang/kernels/jit/csrc/deepselect/vendor/cuda_kernels/config.h @@ -0,0 +1,27 @@ +#pragma once + +#include + +template< + typename ValueT_, typename OutIdxT_, + bool sorted_value_, bool sorted_index_, bool return_value_, + uint32_t max_topk_, + uint32_t num_threads_, uint32_t target_occupancy_, + uint32_t elements_per_round_, uint32_t reconstruct_threshold_, uint32_t tma_buffer_depth_, uint32_t elements_per_segment_ = 512, + uint32_t cluster_size_ = 1 +> +struct TopkSelectConfig { + using ValueT = ValueT_; + using OutIdxT = OutIdxT_; + static constexpr bool sorted_value = sorted_value_; + static constexpr bool sorted_index = sorted_index_; + static constexpr bool return_value = return_value_; + static constexpr uint32_t max_topk = max_topk_; + static constexpr uint32_t num_threads = num_threads_; + static constexpr uint32_t target_occupancy = target_occupancy_; // Number of CTAs per SM + static constexpr uint32_t elements_per_round = elements_per_round_; + static constexpr uint32_t reconstruct_threshold = reconstruct_threshold_; + static constexpr uint32_t tma_buffer_depth = tma_buffer_depth_; + static constexpr uint32_t elements_per_segment = elements_per_segment_; + static constexpr uint32_t cluster_size = cluster_size_; // (1 = not a cluster variant) +}; diff --git a/python/sglang/kernels/jit/csrc/deepselect/vendor/cuda_kernels/utils.cuh b/python/sglang/kernels/jit/csrc/deepselect/vendor/cuda_kernels/utils.cuh new file mode 100644 index 0000000000..44430fe78f --- /dev/null +++ b/python/sglang/kernels/jit/csrc/deepselect/vendor/cuda_kernels/utils.cuh @@ -0,0 +1,54 @@ +#pragma once + +#include + +template +__device__ __forceinline__ T warp_level_inclusive_prefix_sum(T x, uint32_t lane_idx) { + static_assert(sizeof(T) == 4); + #pragma unroll + for (uint32_t i = 1; i <= 16; i <<= 1) { + asm volatile ( + "{\n" + ".reg .pred p;\n" + ".reg .b32 t;\n" + "shfl.sync.up.b32 t|p, %0, %1, 0, 0xffffffff;\n" + "@p add.u32 %0, %0, t;\n" + "}\n" + : "+r"(x) + : "r"(i) + ); + } + return x; +} + +template +__device__ __forceinline__ T warp_level_exclusive_prefix_sum(T x, uint32_t lane_idx) { + T inclusive_prefix_sum = warp_level_inclusive_prefix_sum(x, lane_idx); + return inclusive_prefix_sum - x; +} + +template +__device__ __forceinline__ T warp_level_inclusive_suffix_sum(T x, uint32_t lane_idx) { + static_assert(sizeof(T) == 4); + #pragma unroll + for (uint32_t i = 1; i <= 16; i <<= 1) { + asm volatile ( + "{\n" + ".reg .pred p;\n" + ".reg .b32 t;\n" + "shfl.sync.down.b32 t|p, %0, %1, 31, 0xffffffff;\n" + "@p add.u32 %0, %0, t;\n" + "}\n" + : "+r"(x) + : "r"(i) + ); + } + return x; +} + + +template +__device__ __forceinline__ T warp_level_exclusive_suffix_sum(T x, uint32_t lane_idx) { + T inclusive_suffix_sum = warp_level_inclusive_suffix_sum(x, lane_idx); + return inclusive_suffix_sum - x; +} diff --git a/python/sglang/kernels/jit/csrc/deepselect/vendor/cuda_kernels/v3/topk_select.cuh b/python/sglang/kernels/jit/csrc/deepselect/vendor/cuda_kernels/v3/topk_select.cuh new file mode 100644 index 0000000000..483000aa7d --- /dev/null +++ b/python/sglang/kernels/jit/csrc/deepselect/vendor/cuda_kernels/v3/topk_select.cuh @@ -0,0 +1,147 @@ +/* +TopK select kernel bf16 common variant. + +Usage caveats: + - vocab_size must be < 2^23. + - Do NOT assume that the kernel will select a prefix index on ties. +*/ + +#pragma once + +#include +#include +#include + +#include "structs.h" +#include "cuda_kernels/utils.cuh" +#include "cuda_kernels/common_parts.cuh" + +namespace topk_select_bf16_normal { + +template +class TopkSelectKernelBF16Normal : public topk_select_common::TopkSelectKernelBF16Base { + using BF16Base = topk_select_common::TopkSelectKernelBF16Base; +public: + using ValueT = typename BF16Base::ValueT; + using OutIdxT = typename BF16Base::OutIdxT; + using TmaParams = typename BF16Base::TmaParams; + using SharedMemoryPlanBase = typename BF16Base::SharedMemoryPlanBase; + using EpilogueT = typename BF16Base::EpilogueT; + using BF16Base::NUM_THREADS; + using BF16Base::MAX_TOPK; + using BF16Base::NUM_ELEMS_PER_SEG; + using BF16Base::NUM_ELEMS_PER_ROUND; + using BF16Base::NUM_SEGS_IN_INIT_WINDOW; + using BF16Base::NUM_INIT_ROUNDS_MAX; + using BF16Base::NUM_TAIL_ELEMS; + using BF16Base::NUM_TAIL_SEGS; + using BF16Base::NUM_SEGS_PER_ROUND; + + static_assert(Config::target_occupancy >= 1 && Config::target_occupancy <= 8); + static_assert(MAX_TOPK == 512 || MAX_TOPK == 1024 || MAX_TOPK == 4096); + static constexpr bool HAS_PARTIAL_ROUNDS = NUM_SEGS_PER_ROUND > NUM_TAIL_SEGS; + struct SharedMemoryPlan : SharedMemoryPlanBase {}; + + static __device__ __forceinline__ + void topk_select_kernel_devfunc(const TopkSelectArgs &args, const TmaParams &tma_params) { + uint32_t batch_idx = blockIdx.x; + uint32_t end_vocab_idx = args.end_ptr == nullptr ? args.vocab_size : __ldg(args.end_ptr + batch_idx); + + extern __shared__ CUTE_ALIGNAS(1024) char wksp_buf[]; + SharedMemoryPlan &smem = *reinterpret_cast(wksp_buf); + + uint32_t warp_idx = cutlass::canonical_warp_idx_sync(); + uint32_t lane_idx = threadIdx.x % 32; + + if (end_vocab_idx <= args.topk) { + EpilogueT::template topk_select_epilogue( + (ValueT*)smem.surviving_topk_pairs[0], + (uint32_t*)(smem.surviving_topk_pairs[1]), + args, + batch_idx, end_vocab_idx, warp_idx, + *reinterpret_cast(smem.incoming_topk_pairs) + ); + return; + } + + uint32_t survivor_buf_idx = 0; // Current candidate buffer (A/B, swapped by reconstruct) + bool have_nan = false; + + BF16Base::init_shared_memory(smem, warp_idx); + + uint32_t num_input_segs = ku::ceil_div(end_vocab_idx, (uint32_t)NUM_ELEMS_PER_SEG); + uint32_t num_perm_segs = num_input_segs <= NUM_SEGS_IN_INIT_WINDOW ? 0u : (num_input_segs - 1) / NUM_TAIL_SEGS * NUM_TAIL_SEGS; // Skip permutation if the total number of segs is small enough + uint32_t num_perm_elems = num_perm_segs * NUM_ELEMS_PER_SEG; + uint32_t num_tail_elems_padded = num_perm_segs != 0 ? (uint32_t)NUM_TAIL_ELEMS : num_input_segs * NUM_ELEMS_PER_SEG; + uint32_t num_rounds = ku::ceil_div(num_tail_elems_padded + num_perm_elems, (uint32_t)NUM_ELEMS_PER_ROUND); + uint32_t num_init_rounds = min(num_rounds, (uint32_t)NUM_INIT_ROUNDS_MAX); + + BF16Base::template scan_segs( + tma_params, smem, + batch_idx, end_vocab_idx, args.topk, + warp_idx, lane_idx, + num_perm_segs, + 0, + num_perm_segs, + num_tail_elems_padded, + end_vocab_idx - num_perm_elems, + survivor_buf_idx, + have_nan, + [&](uint32_t m) { + if constexpr (HAS_PARTIAL_ROUNDS) { + return (num_init_rounds + m) * NUM_SEGS_PER_ROUND + warp_idx < NUM_TAIL_SEGS + num_perm_segs; + } else { + return true; + } + } + ); + + // `__syncthreads_or` is also the barrier that makes the last `reconstruct`'s writes visible before + // `stage_output_and_epilogue` reads the survivor buffer. + have_nan = __syncthreads_or(have_nan) != 0; + if (have_nan) { + BF16Base::take_action_when_have_nan(args, batch_idx); + return; + } + + BF16Base::template stage_output_and_epilogue( + smem, *reinterpret_cast(smem.incoming_topk_pairs), args, + batch_idx, end_vocab_idx, survivor_buf_idx, warp_idx, lane_idx); + } +}; + +template +__launch_bounds__(Kernel::NUM_THREADS, Kernel::TARGET_OCCUPANCY, 1) +__global__ void topk_kernel(__grid_constant__ const TopkSelectArgs args, __grid_constant__ const typename Kernel::TmaParams tma_params) { + Kernel::topk_select_kernel_devfunc(args, tma_params); +} + +template +void run_topk_select_kernel(const TopkSelectArgs &args) { + KU_ASSERT(args.sorted_value == Config::sorted_value, "Dispatch failure"); + KU_ASSERT(args.sorted_index == Config::sorted_index, "Dispatch failure"); + KU_ASSERT(args.return_value == Config::return_value, "Dispatch failure"); + static_assert(cute::is_same_v); + KU_ASSERT(args.vocab_size < MAX_VOCAB_SIZE, "`vocab_size` is too big"); + + using Kernel = TopkSelectKernelBF16Normal; + KU_ASSERT(args.topk <= Kernel::MAX_TOPK, "topk is too large. Maximum allowed: %d\n", Kernel::MAX_TOPK); + static_assert(INPUT_STRIDE_ALIGNMENT_REQUIREMENT % 16 == 0); + + auto kernel = topk_kernel; + constexpr size_t smem_size = sizeof(typename Kernel::SharedMemoryPlan); + KU_ASSERT(smem_size * Kernel::TARGET_OCCUPANCY <= args.shared_memory_size_per_sm); + KU_CUDA_CHECK(cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, smem_size)); + + KU_ASSERT(args.stride_input_batch % 8 == 0, "stride_input_batch must be 16B-aligned"); + typename Kernel::TmaParams tma_params = {Kernel::make_topk_tensor_map(args)}; + + ku::launch_kernel(ku::KernelLaunchConfig { + dim3(args.batch_size), + dim3(Kernel::NUM_THREADS), + smem_size, + args.stream + }, kernel, args, tma_params); +} + +} // topk_select_bf16_normal diff --git a/python/sglang/kernels/jit/csrc/deepselect/vendor/cuda_kernels/v3_cluster/topk_select.cuh b/python/sglang/kernels/jit/csrc/deepselect/vendor/cuda_kernels/v3_cluster/topk_select.cuh new file mode 100644 index 0000000000..47e8fba698 --- /dev/null +++ b/python/sglang/kernels/jit/csrc/deepselect/vendor/cuda_kernels/v3_cluster/topk_select.cuh @@ -0,0 +1,305 @@ +/* +TopK select kernel bf16 cluster variant. Use multiple CTAs to calculate one sequence to avoid wave quantization + +Algorithm: +- Each CTA computes its local top-k pairs +- Each CTA sends its local top-k pairs to CTA0 +- CTA0 selects top-k from those pairs +*/ + +#pragma once + +#include +#include +#include +#include + +#include "structs.h" +#include "cuda_kernels/utils.cuh" +#include "cuda_kernels/common_parts.cuh" + +namespace topk_select_bf16_cluster { + +CUTE_DEVICE +static void st_async_32b(uint32_t dst_addr, const uint32_t& data, transac_bar_t &mbar) { + uint32_t mbar_addr = cute::cast_smem_ptr_to_uint(&mbar); + asm volatile ( + "st.async.weak.shared::cluster.mbarrier::complete_tx::bytes.s32 [%0], {%1}, [%2]; \n" + : + : "r"(dst_addr), "r"(data), "r"(mbar_addr) + ); +} + +template +class TopkSelectKernelBF16Cluster : public topk_select_common::TopkSelectKernelBF16Base { + using BF16Base = topk_select_common::TopkSelectKernelBF16Base; +public: + using ValueT = typename BF16Base::ValueT; + using OutIdxT = typename BF16Base::OutIdxT; + using TmaParams = typename BF16Base::TmaParams; + using SharedMemoryPlanBase = typename BF16Base::SharedMemoryPlanBase; + using EpilogueT = typename BF16Base::EpilogueT; + using BF16Base::NUM_THREADS; + using BF16Base::MAX_TOPK; + using BF16Base::NUM_EXTRA_SLOTS; + using BF16Base::NUM_ELEMS_PER_SEG; + using BF16Base::NUM_ELEMS_PER_ROUND; + using BF16Base::NUM_SEGS_IN_INIT_WINDOW; + using BF16Base::NUM_INIT_ROUNDS_MAX; + using BF16Base::NUM_TAIL_ELEMS; + using BF16Base::NUM_TAIL_SEGS; + using BF16Base::NUM_SEGS_PER_ROUND; + + static_assert(Config::target_occupancy == 1); + static_assert(MAX_TOPK == 512 || MAX_TOPK == 1024); + static_assert(Config::cluster_size == 8 || Config::cluster_size == 16); + static_assert(NUM_ELEMS_PER_ROUND * sizeof(ValueT) % 1024 == 0); + static_assert((uint64_t)(MAX_VOCAB_SIZE / NUM_ELEMS_PER_SEG) * (MAX_VOCAB_SIZE / NUM_ELEMS_PER_SEG) + + BF16Base::PERM_ADD_BASE <= 0xFFFFFFFFull); + static constexpr uint32_t CLUSTER_SIZE = Config::cluster_size; + static constexpr uint32_t NUM_GATHER_PAIRS = Config::cluster_size * MAX_TOPK; + static constexpr uint32_t NUM_GATHER_UNITS = NUM_GATHER_PAIRS / 2; + static constexpr uint32_t NUM_UINT32_GATHER_PER_THREAD = ((NUM_GATHER_UNITS + NUM_THREADS - 1) / NUM_THREADS) | 1u; + struct SharedMemoryPlanBF16Cluster : SharedMemoryPlanBase { + kerutils::transac_bar_t gather_val_bar; + kerutils::transac_bar_t gather_bar; + CUTE_ALIGNAS(16) uint32_t gathered_num_survivors[Config::cluster_size]; + }; + static_assert(NUM_GATHER_UNITS % NUM_THREADS == 0); + static_assert(NUM_GATHER_PAIRS <= 0xFFFF); + static_assert(2 * NUM_UINT32_GATHER_PER_THREAD <= 256); + + static __device__ __forceinline__ void topk_select_kernel_devfunc(const TopkSelectArgs &args, const TmaParams &tma_params) { + uint32_t rank_in_cluster = blockIdx.x; + uint32_t batch_idx = blockIdx.y; + uint32_t end_vocab_idx = args.end_ptr == nullptr ? args.vocab_size : __ldg(args.end_ptr + batch_idx); + + extern __shared__ CUTE_ALIGNAS(1024) char wksp_buf[]; + SharedMemoryPlanBF16Cluster &smem = *reinterpret_cast(wksp_buf); + + uint32_t warp_idx = cutlass::canonical_warp_idx_sync(); + uint32_t lane_idx = threadIdx.x % 32; + + if (end_vocab_idx <= args.topk) { + if (rank_in_cluster != 0) { + return; + } + EpilogueT::template topk_select_epilogue( + (ValueT*)smem.surviving_topk_pairs[0], + (uint32_t*)(smem.surviving_topk_pairs[1]), + args, + batch_idx, end_vocab_idx, warp_idx, + *reinterpret_cast(smem.incoming_topk_pairs) + ); + return; + } + + uint32_t survivor_buf_idx = 0; // current candidate buffer (A/B, swapped by reconstruct) + + BF16Base::init_shared_memory(smem, warp_idx, [&]{ + smem.gather_val_bar.init(1); + smem.gather_bar.init(1); + }); + + // Still, divide the whole input into the "perm" part and the "tail" part + // The "tail" part contains the last 1 ~ NUM_TAIL_ELEMS elements and the "perm" part has the other elements. + uint32_t num_input_segs = ku::ceil_div(end_vocab_idx, (uint32_t)NUM_ELEMS_PER_SEG); + uint32_t num_perm_segs = num_input_segs <= NUM_SEGS_IN_INIT_WINDOW ? 0u : (num_input_segs - 1) / NUM_TAIL_SEGS * NUM_TAIL_SEGS; + uint32_t num_perm_elems = num_perm_segs * NUM_ELEMS_PER_SEG; + uint32_t num_tail_elems_padded = num_perm_segs != 0 ? (uint32_t)NUM_TAIL_ELEMS : num_input_segs * NUM_ELEMS_PER_SEG; + + // The "tail" part is handled by CTA0 in the cluster + // "local" means the current CTA (in the cluster) + uint32_t num_local_tail_elems_padded = rank_in_cluster == 0 ? num_tail_elems_padded : 0u; + uint32_t num_local_tail_segs = num_local_tail_elems_padded / NUM_ELEMS_PER_SEG; + uint32_t num_local_tail_elems = rank_in_cluster == 0 ? end_vocab_idx - num_perm_elems : 0u; + + // rank r takes the r-th contiguous interval of the row's visit order, with equal visit elements per rank + uint32_t num_total_elems_padded = num_tail_elems_padded + num_perm_elems; + // Rank r's slice of the padded row starts at r * num_total_elems_padded / cluster_size; the tail is + // rank 0's alone, so clamp that boundary into the perm part (element index within the perm part). + auto perm_boundary = [&](uint32_t rank) { + static_assert((uint64_t)Config::cluster_size * (MAX_VOCAB_SIZE + NUM_TAIL_ELEMS) <= 0xFFFFFFFFull); + uint32_t boundary = rank * num_total_elems_padded / Config::cluster_size; + return boundary > num_tail_elems_padded ? boundary - num_tail_elems_padded : 0u; + }; + uint32_t local_start_seg_idx = perm_boundary(rank_in_cluster) / NUM_ELEMS_PER_SEG; + uint32_t local_end_seg_idx = perm_boundary(rank_in_cluster + 1) / NUM_ELEMS_PER_SEG; + uint32_t num_local_perm_segs = local_end_seg_idx - local_start_seg_idx; + uint32_t num_local_elems = num_local_tail_elems_padded + num_local_perm_segs * NUM_ELEMS_PER_SEG; + + uint32_t num_local_rounds = ku::ceil_div(num_local_elems, (uint32_t)NUM_ELEMS_PER_ROUND); + uint32_t num_local_init_rounds = min(num_local_rounds, (uint32_t)NUM_INIT_ROUNDS_MAX); + + uint32_t num_local_segs = num_local_tail_segs + num_local_perm_segs; + uint32_t used_segs = num_local_init_rounds * NUM_SEGS_PER_ROUND; + uint32_t rem_segs = num_local_segs > used_segs ? num_local_segs - used_segs : 0u; + uint32_t warp_rem_segs = warp_idx < rem_segs ? rem_segs - warp_idx : 0u; + uint32_t warp_round_limit = ku::ceil_div(warp_rem_segs, (uint32_t)NUM_SEGS_PER_ROUND); + + // Each CTA gets its local top-k elements + bool nan_seen = false; + BF16Base::template scan_segs( + tma_params, smem, + batch_idx, end_vocab_idx, args.topk, warp_idx, lane_idx, + num_perm_segs, local_start_seg_idx, num_local_perm_segs, num_local_tail_elems_padded, num_local_tail_elems, + survivor_buf_idx, + nan_seen, + [&](uint32_t m) { return m < warp_round_limit; } + ); + + // CTA-wide OR; this is also the barrier that makes the scan's writes to the survivor buffer visible to the gather below + nan_seen = __syncthreads_or(nan_seen) != 0; + + // The cluster barrier must not complete before every CTA has finished its own scan: the gather + // below writes into CTA0's shared memory, so a CTA that finished early would otherwise clobber + // CTA0's `tma_load_buf` / `incoming_topk_pairs` while CTA0 is still scanning them. + ku::barrier_cluster_arrive_relaxed(); // Paired with the `barrier_cluster_wait_acquire` below + ku::barrier_cluster_wait_acquire(); + + static_assert(NUM_GATHER_UNITS * sizeof(uint32_t) <= sizeof(smem.incoming_topk_pairs)); + void* val_dst = (void*)(int64_t)cute::set_block_rank( + cute::cast_smem_ptr_to_uint((uint32_t*)smem.incoming_topk_pairs + rank_in_cluster * (MAX_TOPK / 2)), 0); + + static_assert(NUM_GATHER_PAIRS * sizeof(uint64_t) <= sizeof(SharedMemoryPlanBase::tma_load_buf)); + void* dst = (void*)(int64_t)cute::set_block_rank( + cute::cast_smem_ptr_to_uint(reinterpret_cast(smem.tma_load_buf) + rank_in_cluster * MAX_TOPK), 0); + + static_assert(MAX_TOPK*sizeof(ValueT) % 16 == 0); + constexpr uint32_t NUM_VAL_CHUNKS = MAX_TOPK * sizeof(ValueT) / 16; // 16B store + constexpr uint32_t NUM_VAL_CHUNKS_PER_THREAD = ku::ceil_div(NUM_VAL_CHUNKS, NUM_THREADS); + + if (warp_idx == 0 && cute::elect_one_sync()) { + if (rank_in_cluster == 0) { + smem.gather_val_bar.arrive_and_expect_tx((NUM_VAL_CHUNKS * 16 + sizeof(uint32_t)) * CLUSTER_SIZE); + smem.gather_bar.arrive_and_expect_tx((MAX_TOPK * (uint32_t)sizeof(uint64_t)) * CLUSTER_SIZE); + } + st_async_32b( + cute::set_block_rank(cute::cast_smem_ptr_to_uint(smem.gathered_num_survivors + rank_in_cluster), 0), + (uint32_t)nan_seen, + smem.gather_val_bar + ); + } + + // Store values first... so that CTA0 can begin pivot selection earlier + CUTE_UNROLL + for (uint32_t i = 0; i < NUM_VAL_CHUNKS_PER_THREAD; ++i) { + uint32_t c = i * NUM_THREADS + threadIdx.x; + if constexpr (NUM_VAL_CHUNKS % NUM_THREADS != 0) { + if (c >= NUM_VAL_CHUNKS) break; + } + uint32_t vw[4]; + CUTE_UNROLL + for (uint32_t j = 0; j < 4; ++j) { + uint32_t pair2[4]; + topk_select_common::ld_shared<4>(pair2, (const uint32_t*)(smem.surviving_topk_pairs[survivor_buf_idx] + 8 * c + 2 * j)); + vw[j] = __byte_perm(pair2[1], pair2[3], 0x5410); + } + ku::st_async((char*)val_dst + c * sizeof(uint4), make_uint4(vw[0], vw[1], vw[2], vw[3]), + smem.gather_val_bar); + } + + // And then store those (value, index) pairs + static_assert(MAX_TOPK * sizeof(uint64_t) % 16 == 0); + constexpr uint32_t NUM_INDEX_VALUE_CHUNKS = MAX_TOPK * sizeof(uint64_t) / 16; + static_assert(NUM_INDEX_VALUE_CHUNKS % NUM_THREADS == 0); + CUTE_UNROLL + for (uint32_t i = 0; i < NUM_INDEX_VALUE_CHUNKS / NUM_THREADS; ++i) { + uint32_t s = 2 * (i * NUM_THREADS + threadIdx.x); + ulonglong2 pp = *reinterpret_cast(smem.surviving_topk_pairs[survivor_buf_idx] + s); + ku::st_async((char*)dst + s * sizeof(uint64_t), pp, smem.gather_bar); + } + + if (rank_in_cluster != 0) { + return; + } + + // The following code is CTA0 only + smem.gather_val_bar.wait(0); + BF16Base::clear_reconstruct_histograms(smem, threadIdx.x); + __syncthreads(); + + static_assert(Config::cluster_size <= 32); + uint32_t stored_num_survivors = lane_idx < Config::cluster_size ? smem.gathered_num_survivors[lane_idx] : 0u; + nan_seen |= stored_num_survivors; + + uint32_t unit_base = threadIdx.x * NUM_UINT32_GATHER_PER_THREAD; + uint32_t num_my_units = unit_base < NUM_GATHER_UNITS ? min((uint32_t)NUM_UINT32_GATHER_PER_THREAD, NUM_GATHER_UNITS - unit_base) : 0u; + + const uint32_t *gather_vals = (const uint32_t*)smem.incoming_topk_pairs; + nv_bfloat162 values[NUM_UINT32_GATHER_PER_THREAD]; + CUTE_UNROLL + for (uint32_t i = 0; i < NUM_UINT32_GATHER_PER_THREAD; i++) { + if (i == num_my_units) break; + values[i] = topk_select_common::u32_to_bf16x2(gather_vals[unit_base + i]); + } + BF16Base::histogram_radix_msb(smem.reconstruct_bucket_counter[0], values, num_my_units); + __syncthreads(); + + auto [pivot_value_x2_bits, out_prefix, eq_quota, cnt_nan] = + BF16Base::template compute_pivot_and_quota(args.topk, values, num_my_units, warp_idx, lane_idx, smem); + nan_seen |= cnt_nan != 0; + + smem.gather_bar.wait(0); + __syncthreads(); + + uint32_t out_ptr = cute::cast_smem_ptr_to_uint(smem.surviving_topk_pairs[survivor_buf_idx ^ 1]) + out_prefix * (uint32_t)sizeof(uint64_t); + uint32_t gather_pairs_base = cute::cast_smem_ptr_to_uint(smem.tma_load_buf); + CUTE_UNROLL + for (uint32_t m = 0; m < NUM_UINT32_GATHER_PER_THREAD; m++) { + if (m == num_my_units) break; + BF16Base::template copy_selected_pairs_to_survivor(out_ptr, eq_quota, values[m], pivot_value_x2_bits, gather_pairs_base + 2 * (unit_base + m) * (uint32_t)sizeof(uint64_t)); + } + survivor_buf_idx ^= 1; + + nan_seen = __syncthreads_or(nan_seen) != 0; + if (nan_seen) { + BF16Base::take_action_when_have_nan(args, batch_idx); + return; + } + + BF16Base::template stage_output_and_epilogue( + smem, *reinterpret_cast(smem.incoming_topk_pairs), args, + batch_idx, end_vocab_idx, survivor_buf_idx, warp_idx, lane_idx); + } +}; + +template +__launch_bounds__(Kernel::NUM_THREADS, Kernel::TARGET_OCCUPANCY, Kernel::CLUSTER_SIZE) +__global__ void topk_kernel(__grid_constant__ const TopkSelectArgs args, __grid_constant__ const typename Kernel::TmaParams tma_params) { + Kernel::topk_select_kernel_devfunc(args, tma_params); +} + +template +void run_topk_select_kernel(const TopkSelectArgs &args) { + KU_ASSERT(args.sorted_value == Config::sorted_value, "Dispatch failure"); + KU_ASSERT(args.sorted_index == Config::sorted_index, "Dispatch failure"); + KU_ASSERT(args.return_value == Config::return_value, "Dispatch failure"); + static_assert(cute::is_same_v); + KU_ASSERT(args.vocab_size < MAX_VOCAB_SIZE, "`vocab_size` is too big"); + + using Kernel = TopkSelectKernelBF16Cluster; + KU_ASSERT(args.topk <= Kernel::MAX_TOPK, "topk is too large. Maximum allowed: %d\n", Kernel::MAX_TOPK); + static_assert(INPUT_STRIDE_ALIGNMENT_REQUIREMENT % 16 == 0); + + auto kernel = topk_kernel; + constexpr size_t smem_size = sizeof(typename Kernel::SharedMemoryPlanBF16Cluster); + KU_ASSERT(smem_size * Kernel::TARGET_OCCUPANCY <= args.shared_memory_size_per_sm); + KU_CUDA_CHECK(cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, smem_size)); + if constexpr (Kernel::CLUSTER_SIZE > 8) { + KU_CUDA_CHECK(cudaFuncSetAttribute(kernel, cudaFuncAttributeNonPortableClusterSizeAllowed, 1)); + } + + KU_ASSERT(args.stride_input_batch % 8 == 0, "stride_input_batch must be 16B-aligned"); + typename Kernel::TmaParams tma_params = {Kernel::make_topk_tensor_map(args)}; + + ku::launch_kernel(ku::KernelLaunchConfig { + dim3(Kernel::CLUSTER_SIZE, args.batch_size), + dim3(Kernel::NUM_THREADS), + smem_size, + args.stream, + dim3(Kernel::CLUSTER_SIZE, 1, 1) + }, kernel, args, tma_params); +} + +} // topk_select_bf16_cluster diff --git a/python/sglang/kernels/jit/csrc/deepselect/vendor/cuda_kernels/v3_fp32/topk_select.cuh b/python/sglang/kernels/jit/csrc/deepselect/vendor/cuda_kernels/v3_fp32/topk_select.cuh new file mode 100644 index 0000000000..b8f1d40035 --- /dev/null +++ b/python/sglang/kernels/jit/csrc/deepselect/vendor/cuda_kernels/v3_fp32/topk_select.cuh @@ -0,0 +1,601 @@ +/* +TopK select kernel fp32 variant. + +Usage caveats: + - vocab_size must be < 2^23. + - Do NOT assume that the kernel will select a prefix index on ties. +*/ + +#pragma once + +#include +#include +#include + +#include "structs.h" +#include "cuda_kernels/utils.cuh" +#include "cuda_kernels/common_parts.cuh" + +namespace topk_select_fp32 { + +template +class TopkSelectKernelFP32 : public topk_select_common::TopkSelectKernelBase { + using Base = topk_select_common::TopkSelectKernelBase; +public: + using ValueT = typename Base::ValueT; + using OutIdxT = typename Base::OutIdxT; + using TmaParams = typename Base::TmaParams; + using SharedMemoryPlanBase = typename Base::SharedMemoryPlanBase; + using EpilogueT = typename Base::EpilogueT; + using Base::NUM_THREADS; + using Base::NUM_WARPS; + using Base::MAX_TOPK; + using Base::NUM_ELEMS_PER_128b; + using Base::NUM_UINT32_PER_128b; + using Base::NUM_ELEMS_PER_ROUND; + using Base::NUM_ELEMS_PER_THREAD_PER_ROUND; + using Base::ELEMS_PER_THREAD_PER_ROUND_MASK; + using Base::NUM_128b_PER_THREAD_PER_ROUND; + using Base::NUM_ELEMS_PER_SEG; + using Base::NUM_TAIL_ELEMS; + using Base::NUM_TAIL_SEGS; + using Base::NUM_ELEMS_IN_INIT_WINDOW; + using Base::NUM_SEGS_IN_INIT_WINDOW; + using Base::NUM_PERM_SEGS_IN_INIT_WINDOW; + using Base::NUM_INIT_ROUNDS_MAX; + using Base::NUM_UINT32_IN_INIT_WINDOW_PER_THREAD; + using Base::NUM_128b_INIT_PER_THREAD; + using Base::NUM_SEGS_PER_ROUND; + using Base::NUM_SEGS_PER_ISSUE_WARP; + using Base::NUM_ISSUE_WARPS; + using Base::NUM_TMA_LOAD_BUFS; + using Base::TMA_PREFETCH_DEPTH; + using Base::RECONSTRUCT_THRESHOLD; + using Base::PERM_ADD_BASE; + using Base::PERM_MUL_PRIME; + using Base::PLACEHOLDER_PAIR; + using Base::NUM_RECONSTRUCT_BUCKETS; + + static constexpr uint32_t PLACEHOLDER = 0xff800000; // -INF + static constexpr uint32_t NEG_INF_BITS = 0xFF800000; + static_assert(MAX_TOPK == 512 || MAX_TOPK == 1024 || MAX_TOPK == 4096); + static constexpr bool HAS_PARTIAL_ROUNDS = NUM_SEGS_PER_ROUND > NUM_TAIL_SEGS; + + struct SharedMemoryPlanFP32 : SharedMemoryPlanBase {}; + + static __device__ __forceinline__ + void append_pair_at(uint32_t &dst_ptr, uint32_t index, uint32_t val_bits) { + *(uint64_t*)__cvta_shared_to_generic(dst_ptr) = (uint64_t)val_bits << 32 | index; + dst_ptr = __float_as_uint(__uint_as_float(dst_ptr) + __uint_as_float(8u)); + } + + static __device__ __forceinline__ void topk_select_kernel_devfunc(const TopkSelectArgs &args, const TmaParams &tma_params) { + uint32_t batch_idx = blockIdx.x; + uint32_t end_vocab_idx = args.end_ptr == nullptr ? args.vocab_size : __ldg(args.end_ptr + batch_idx); + + extern __shared__ CUTE_ALIGNAS(1024) char wksp_buf[]; + SharedMemoryPlanFP32 &smem = *reinterpret_cast(wksp_buf); + + uint32_t warp_idx = cutlass::canonical_warp_idx_sync(); + uint32_t lane_idx = threadIdx.x % 32; + + if (end_vocab_idx <= args.topk) { + EpilogueT::template topk_select_epilogue( + (ValueT*)smem.surviving_topk_pairs[0], + (uint32_t*)(smem.surviving_topk_pairs[1]), + args, + batch_idx, end_vocab_idx, warp_idx, + *reinterpret_cast(smem.incoming_topk_pairs) + ); + return; + } + + uint32_t threshold_bits = NEG_INF_BITS; + uint32_t num_incomers = 0; + uint32_t survivor_buf_idx = 0; // current candidate buffer (A/B, swapped by reconstruct) + bool have_nan = false; + + Base::init_shared_memory(smem, warp_idx); + + uint32_t num_input_segs = ku::ceil_div(end_vocab_idx, (uint32_t)NUM_ELEMS_PER_SEG); + uint32_t num_perm_segs = num_input_segs <= NUM_SEGS_IN_INIT_WINDOW ? 0u : (num_input_segs - 1) / NUM_TAIL_SEGS * NUM_TAIL_SEGS; + uint32_t num_perm_elems = num_perm_segs * NUM_ELEMS_PER_SEG; + uint32_t num_local_tail_elems_padded = num_perm_segs != 0 ? (uint32_t)NUM_TAIL_ELEMS : num_input_segs * NUM_ELEMS_PER_SEG; + uint32_t num_rounds = ku::ceil_div(num_local_tail_elems_padded + num_perm_elems, (uint32_t)NUM_ELEMS_PER_ROUND); + uint32_t num_init_rounds = min(num_rounds, (uint32_t)NUM_INIT_ROUNDS_MAX); + ValueT *init_buf = (ValueT*)smem.incoming_topk_pairs; // aliased + + uint32_t perm_len = max(num_perm_segs, 1u); + uint32_t perm_mul = PERM_MUL_PRIME % perm_len; + static_assert(NUM_SEGS_PER_ROUND == NUM_SEGS_PER_ISSUE_WARP * NUM_ISSUE_WARPS); + __shared__ bool should_select_whole_bucket_shared; + + auto find_pivot_in_histogram = [&](uint32_t topk, const uint32_t *bucket_counter) { + bool should_select_whole_bucket = + Base::template find_pivot_in_histogram(smem, bucket_counter, topk, lane_idx); + if (threadIdx.x == 0) { + should_select_whole_bucket_shared = should_select_whole_bucket; + } + }; + // First radix pass: histogram the most significant byte of each distorted + // fp32 value. Used to locate the top-k bucket prefix. + uint32_t hist_base = cute::cast_smem_ptr_to_uint(smem.reconstruct_bucket_counter[0]); + // mad forms the bucket address and red.shared.add does the increment: no return value, one address add. + auto histogram_radix_msb_one = [&](uint32_t value_bits) { + uint32_t addr; + asm volatile ("mad.lo.u32 %0, %1, 4, %2; red.shared.add.u32 [%0], 1;" + : "=r"(addr) + : "r"(topk_select_common::distort(value_bits) >> 24), "r"(hist_base) + : "memory"); + }; + auto histogram_radix_msb = [&](const uint32_t (&vals)[N], uint32_t m_end) { + CUTE_UNROLL + for (uint32_t i = 0; i < N; i++) { + if (i == m_end) break; + histogram_radix_msb_one(vals[i]); + } + }; + struct PivotAndQuota { + uint32_t pivot_value_bits; + uint32_t start_pos_in_collector; + uint32_t eq_quota; + uint32_t cnt_nan; + }; + + auto compute_pivot_and_quota = [&](uint32_t topk, const auto &for_each_value) -> PivotAndQuota { + if (warp_idx == 0) { + find_pivot_in_histogram(topk, smem.reconstruct_bucket_counter[0]); + } + __syncthreads(); + // `pivot_prefix` tracks the pivot's leading bits in the *undistorted* (raw) domain. + // Inside a round every element that survived the previous round must has the pivot's sign, so the distortion rule (to xor what) is the same. + // So we can optimize scanning & picking & distorting into "comparing the raw leading bits" and "xor-ing `neg_mask`" + uint32_t pivot_prefix_dist = smem.reconstruct_pivot_bucket; // distorted top 8 bits + bool pivot_negative = pivot_prefix_dist < 0x80u; // distorted >= 0x80 <=> positive + uint32_t neg_mask = pivot_negative ? 0xFFu : 0u; + uint32_t pivot_prefix = pivot_prefix_dist ^ (pivot_negative ? 0xFFu : 0x80u); + uint32_t num_elem_should_select_in_pivot_bucket = smem.reconstruct_num_should_select; + bool should_select_whole_bucket = should_select_whole_bucket_shared; // whether round 1 is already a clean boundary + uint32_t rounds_done = 1; + + // Early exit: once topk lands exactly on a bucket's lower edge, the pivot is that bucket's floor. + CUTE_UNROLL + for (uint32_t r = 1; r < 4; ++r) { + if (should_select_whole_bucket) break; + uint32_t *dst = smem.reconstruct_bucket_counter[r & 1]; + uint32_t *clr = smem.reconstruct_bucket_counter[(r & 1) ^ 1]; + uint32_t dst_base = cute::cast_smem_ptr_to_uint(dst); + for_each_value([&](uint32_t value_bits) { + // Elements whose prefix differs from the pivot still add, but to this warp's own sink slot. This let PTXAS generate ATOMS.INC instructions which is faster + bool match = (value_bits >> (32 - 8 * r)) == pivot_prefix; + // bfe extracts the next raw byte in one instruction (ptxas otherwise emits shf + lop3) + uint32_t bucket_raw, bucket; + asm volatile ("bfe.u32 %0, %1, %2, 8;" : "=r"(bucket_raw) : "r"(value_bits), "r"(24 - 8 * r)); + bucket = match ? (bucket_raw ^ neg_mask) : NUM_RECONSTRUCT_BUCKETS; + uint32_t addr; + asm volatile ("mad.lo.u32 %0, %1, 4, %2; red.shared.add.u32 [%0], 1;" + : "=r"(addr) : "r"(bucket), "r"(dst_base) : "memory"); + }); + constexpr uint32_t NUM_BUCKET_CLEAR_128b = NUM_RECONSTRUCT_BUCKETS / 4; + static_assert(NUM_THREADS >= NUM_BUCKET_CLEAR_128b); + if (threadIdx.x < NUM_BUCKET_CLEAR_128b) { + reinterpret_cast<__int128_t*>(clr)[threadIdx.x] = (__int128_t)0ull; + } + __syncthreads(); + if (warp_idx == 0) { + find_pivot_in_histogram(num_elem_should_select_in_pivot_bucket, dst); + } + __syncthreads(); + // The histogram is keyed by distorted bytes, so undo the sign map on the bucket id. + pivot_prefix = (pivot_prefix << 8) | (smem.reconstruct_pivot_bucket ^ neg_mask); + num_elem_should_select_in_pivot_bucket = smem.reconstruct_num_should_select; + should_select_whole_bucket = should_select_whole_bucket_shared; + rounds_done++; + } + // Take the lowest element in the selected bucket + // Be careful that `pivot` may be negative, and we may have to fill the lower bits with 1 + uint32_t pivot_low_shift = (4 - rounds_done) * 8; + uint32_t pivot_value_bits = pivot_prefix << pivot_low_shift; + if (pivot_negative) { + pivot_value_bits |= (1u << pivot_low_shift) - 1u; + } + float pivot_value = __uint_as_float(pivot_value_bits); + + // set.xx.f32.f32 yields 1.0/0.0, so the counts accumulate on the FP pipe instead of the + // (busiest) integer pipe; the counts stay exact because FP32 can represent every integer within 0 ~ 2**23 + float gt_accum = 0.0f, eq_accum = 0.0f; + float nan_accum = 0.0f; + for_each_value([&](uint32_t value_bits) { + float v = __uint_as_float(value_bits); + asm volatile ( + "{\n" + ".reg .f32 g, e;\n" + "set.gt.f32.f32 g, %3, %4;\n" + "add.f32 %0, %0, g;\n" + "set.eq.f32.f32 e, %3, %4;\n" + "add.f32 %1, %1, e;\n" + "min.NaN.f32 %2, %2, %3;\n" // We use `min.NaN` for NaN detection + "}\n" + : "+f"(gt_accum), "+f"(eq_accum), "+f"(nan_accum) + : "f"(v), "f"(pivot_value)); + }); + uint32_t cnt_gt = (uint32_t)gt_accum; + uint32_t cnt_eq = (uint32_t)eq_accum; + float nan_flag; + asm ("set.nan.f32.f32 %0, %1, %1;" : "=f"(nan_flag) : "f"(nan_accum)); + uint32_t cnt_nan = (uint32_t)nan_flag; + + static_assert(NUM_WARPS <= NUM_RECONSTRUCT_BUCKETS); + uint32_t *eq_pass_warp_cnt = smem.reconstruct_bucket_counter[0]; + auto eqgt = Base::compute_equal_quota_and_prefix(cnt_gt, cnt_eq, topk, warp_idx, lane_idx, eq_pass_warp_cnt); + return {pivot_value_bits, eqgt.start_pos_in_collector, eqgt.eq_quota, cnt_nan}; + }; + + auto reconstruct = [&](uint32_t cur_extra_len) -> uint32_t { + uint32_t tid = threadIdx.x; + Base::clear_reconstruct_histograms(smem, tid); + + uint32_t padded_extra_len = (cur_extra_len + 1u) & ~1u; + if (warp_idx == 0 && (cur_extra_len & 1u)) { + smem.incoming_topk_pairs[cur_extra_len] = PLACEHOLDER_PAIR; + } + __syncthreads(); + + uint32_t num_128b = (MAX_TOPK + padded_extra_len) / 2; + uint32_t num_128b_per_thread = ((num_128b + NUM_THREADS - 1) / NUM_THREADS) | 1u; // force odd => bank-conflict-free smem gathers + uint32_t b128_base = tid * num_128b_per_thread; + uint32_t num_my_128b = b128_base < num_128b ? min(num_128b_per_thread, num_128b - b128_base) : 0u; + + auto b128_to_pair_addr = [&](uint32_t u) -> uint32_t { + // The first MAX_TOPK/2 b128 words live in the current candidate buffer, the rest in the incoming region + return u < MAX_TOPK / 2 + ? cute::cast_smem_ptr_to_uint(smem.surviving_topk_pairs[survivor_buf_idx] + 2 * u) + : cute::cast_smem_ptr_to_uint(smem.incoming_topk_pairs + (2 * u - MAX_TOPK)); + }; + + // One b128 word is 16 bytes = 2 pairs {index0, value0, index1, value1}; + constexpr uint32_t NUM_128b_MAX = (MAX_TOPK + RECONSTRUCT_THRESHOLD + NUM_ELEMS_PER_ROUND) / 2; + constexpr uint32_t NUM_128b_PER_THREAD = ((NUM_128b_MAX + NUM_THREADS - 1) / NUM_THREADS) | 1u; // | 1u since we may | 1u when generating `num_128b_per_thread` + auto for_each_value = [&](const auto &fn) { + CUTE_UNROLL + for (uint32_t i = 0; i < NUM_128b_PER_THREAD; i++) { + if (i == num_my_128b) break; + uint32_t pair2[4]; + topk_select_common::ld_shared<4>(pair2, (const uint32_t*)b128_to_pair_addr(b128_base + i)); + fn(pair2[1]); + fn(pair2[3]); + } + }; + + for_each_value(histogram_radix_msb_one); + __syncthreads(); + + uint32_t topk = args.topk; + auto [pivot_value_bits, start_pos_in_collector, eq_quota, cnt_nan] = compute_pivot_and_quota(topk, for_each_value); + // Every NaN the main loop collected is part of the buffer this census just walked (the hit test + // is `.gtu`, so NaN always becomes an incomer); the CTA-wide OR happens at the end. + have_nan |= cnt_nan != 0; + + { + uint32_t out_ptr = cute::cast_smem_ptr_to_uint(smem.surviving_topk_pairs[survivor_buf_idx ^ 1]) + start_pos_in_collector * (uint32_t)sizeof(uint64_t); + float pivot_value = __uint_as_float(pivot_value_bits); + CUTE_UNROLL + for (uint32_t i = 0; i < NUM_128b_PER_THREAD; i++) { + if (i == num_my_128b) break; + uint32_t pair2[4]; + topk_select_common::ld_shared<4>(pair2, (const uint32_t*)b128_to_pair_addr(b128_base + i)); + float v0 = __uint_as_float(pair2[1]); + float v1 = __uint_as_float(pair2[3]); + bool t0 = v0 == pivot_value && eq_quota != 0; + eq_quota -= t0; + if (v0 > pivot_value || t0) { append_pair_at(out_ptr, pair2[0], pair2[1]); } + bool t1 = v1 == pivot_value && eq_quota != 0; + eq_quota -= t1; + if (v1 > pivot_value || t1) { append_pair_at(out_ptr, pair2[2], pair2[3]); } + } + } + + survivor_buf_idx ^= 1; + return pivot_value_bits; + }; + + uint32_t tma_permuted_segment_stride = NUM_ISSUE_WARPS * perm_mul % perm_len; + uint32_t next_tma_permuted_segment = Base::get_permuted_seg_idx(warp_idx, perm_len, perm_mul); + auto issue_tma_loads_for_round = [&](uint32_t idx) { + Base::template issue_tma_loads_for_round( + smem, tma_params.tensor_map, batch_idx, end_vocab_idx, + num_perm_segs, idx, next_tma_permuted_segment, tma_permuted_segment_stride, warp_idx); + }; + + if (cute::elect_one_sync()) { + issue_tma_loads_for_round.template operator()(0); + CUTE_UNROLL + for (uint32_t i = 1; i < NUM_INIT_ROUNDS_MAX; i++) { + if (i < num_init_rounds) { + issue_tma_loads_for_round.template operator()(i); + } + } + { + uint32_t num_main_prefetch = num_rounds - num_init_rounds; + CUTE_UNROLL + for (uint32_t i = 0; i < TMA_PREFETCH_DEPTH; i++) { + if (i < num_main_prefetch) { + issue_tma_loads_for_round.template operator()(i); + } + } + } + } + + { // INIT + uint32_t num_elems_in_init_window_padded = num_local_tail_elems_padded + min(num_perm_segs, (uint32_t)NUM_PERM_SEGS_IN_INIT_WINDOW) * NUM_ELEMS_PER_SEG; + uint32_t num_128b_in_init_window_padded = num_elems_in_init_window_padded / NUM_ELEMS_PER_128b; + uint32_t cnt_floor = num_128b_in_init_window_padded / NUM_THREADS; + uint32_t cnt_rem = num_128b_in_init_window_padded % NUM_THREADS; + uint32_t my_elem_start_idx = (threadIdx.x * cnt_floor + min(threadIdx.x, cnt_rem)) * NUM_ELEMS_PER_128b; + uint32_t num_my_elems = NUM_UINT32_PER_128b * cnt_floor + (threadIdx.x < cnt_rem ? NUM_UINT32_PER_128b : 0u); + uint32_t num_local_tail_elems = end_vocab_idx - num_perm_elems; + + if (num_perm_segs != 0) { + Base::fill_padded_tail_segments(init_buf, end_vocab_idx - num_perm_elems); + __syncthreads(); + } + + CUTE_UNROLL + for (uint32_t i = 0; i < NUM_INIT_ROUNDS_MAX; i++) { + if (i == num_init_rounds) break; + smem.init_full_bar[i].wait(0); + if (num_local_tail_elems % NUM_ELEMS_PER_SEG != 0 && i == num_local_tail_elems / NUM_ELEMS_PER_ROUND) { + uint32_t box_end = ku::ceil_div(num_local_tail_elems, (uint32_t)NUM_ELEMS_PER_SEG) * NUM_ELEMS_PER_SEG; + for (uint32_t e = num_local_tail_elems + threadIdx.x; e < box_end; e += NUM_THREADS) { + init_buf[Base::sw_elem(e)] = __uint_as_float(PLACEHOLDER); + } + __syncthreads(); + } + uint32_t my_values[4 * NUM_128b_PER_THREAD_PER_ROUND]; + uint32_t num_my_values = 0; + CUTE_UNROLL + for (uint32_t k = 0; k < NUM_128b_PER_THREAD_PER_ROUND; ++k) { + uint32_t u = threadIdx.x + (i * NUM_128b_PER_THREAD_PER_ROUND + k) * NUM_THREADS; + if (u < num_128b_in_init_window_padded) { + topk_select_common::ld_shared<4>(my_values + k * 4, (const uint32_t*)(init_buf + Base::sw_b128(u) * NUM_ELEMS_PER_128b)); + num_my_values = 4 * (k + 1); + } + } + histogram_radix_msb(my_values, num_my_values); + } + + uint32_t init_values[NUM_UINT32_IN_INIT_WINDOW_PER_THREAD]; + CUTE_UNROLL + for (uint32_t j = 0; j < NUM_128b_INIT_PER_THREAD; ++j) { + if (j * NUM_UINT32_PER_128b == num_my_elems) break; + uint32_t u0 = my_elem_start_idx / NUM_ELEMS_PER_128b + j; + topk_select_common::ld_shared<4>(init_values + j * NUM_UINT32_PER_128b, (const uint32_t*)(init_buf + Base::sw_b128(u0) * NUM_ELEMS_PER_128b)); + } + + __syncthreads(); // publish the round-1 histogram; also the last read of init_buf + + static_assert(NUM_ELEMS_IN_INIT_WINDOW <= 0xFFFF); + + auto for_each_init_value = [&](const auto &fn) { + CUTE_UNROLL + for (uint32_t i = 0; i < NUM_UINT32_IN_INIT_WINDOW_PER_THREAD; i++) { + if (i == num_my_elems) break; + fn(init_values[i]); + } + }; + auto [pivot_value_bits, start_pos_in_collector, eq_quota, cnt_nan] = compute_pivot_and_quota(args.topk, for_each_init_value); + // The init census walks the thread's whole slice of the window, i.e. every element of the init + // window exactly once, so NaNs inside the window are caught here. + have_nan |= cnt_nan != 0; + + { + uint32_t out_ptr = cute::cast_smem_ptr_to_uint(smem.surviving_topk_pairs[0]) + start_pos_in_collector * (uint32_t)sizeof(uint64_t); + float pivot_value = __uint_as_float(pivot_value_bits); + uint32_t s0 = (max(my_elem_start_idx, num_local_tail_elems_padded) - num_local_tail_elems_padded) / NUM_ELEMS_PER_SEG; + uint32_t perm_state = Base::get_permuted_seg_idx(s0, perm_len, perm_mul); + float b128_base_f = 0.0f; + CUTE_UNROLL + for (uint32_t i = 0; i < NUM_UINT32_IN_INIT_WINDOW_PER_THREAD; i += 2) { + if (i == num_my_elems) break; + if (i % NUM_ELEMS_PER_128b == 0) { + uint32_t g_u = my_elem_start_idx + i; // fp32: word index == element index + if (g_u < num_local_tail_elems_padded) { + b128_base_f = __uint_as_float(num_perm_elems + g_u); + } else { + b128_base_f = __uint_as_float(perm_state * NUM_ELEMS_PER_SEG + g_u % NUM_ELEMS_PER_SEG); + if (g_u % NUM_ELEMS_PER_SEG == NUM_ELEMS_PER_SEG - NUM_ELEMS_PER_128b) { + Base::advance_perm_state(perm_state, perm_mul, perm_len); + } + } + } + // fp32-subnormal FP-pipe int-add (see file-head): index = base + element offset + uint32_t index0 = __float_as_uint(b128_base_f + __uint_as_float(i % NUM_ELEMS_PER_128b)); + uint32_t index1 = __float_as_uint(b128_base_f + __uint_as_float(i % NUM_ELEMS_PER_128b + 1)); + float v0 = __uint_as_float(init_values[i]); + float v1 = __uint_as_float(init_values[i + 1]); + bool t0 = v0 == pivot_value && eq_quota != 0; + eq_quota -= t0; + if (v0 > pivot_value || t0) { append_pair_at(out_ptr, index0, init_values[i]); } + // the quota is re-tested after element 0's decrement + bool t1 = v1 == pivot_value && eq_quota != 0; + eq_quota -= t1; + if (v1 > pivot_value || t1) { append_pair_at(out_ptr, index1, init_values[i + 1]); } + } + } + + threshold_bits = pivot_value_bits; + __syncthreads(); + } + + uint32_t linear_segment_start = + num_init_rounds * NUM_SEGS_PER_ROUND + - NUM_TAIL_SEGS + + warp_idx; + uint32_t current_permuted_segment = Base::get_permuted_seg_idx(linear_segment_start, perm_len, perm_mul); + uint32_t permuted_segment_stride_per_round = NUM_SEGS_PER_ROUND % perm_len * perm_mul % perm_len; + + uint32_t num_main_rounds = num_rounds - num_init_rounds; + // Track the TMA buffer index and its phase incrementally instead of dividing every round. + uint32_t tma_buf_idx = 0; + uint32_t tma_buf_phase = 0; + for (uint32_t main_round_idx = 0; main_round_idx < num_main_rounds; ++main_round_idx) { + // This thread's contiguous chunk starts at logical_elem_offset in this round. + uint32_t logical_elem_offset = threadIdx.x * NUM_ELEMS_PER_THREAD_PER_ROUND; + // Offset inside the 512-element segment; used later to rebuild original indices. + uint32_t offset_in_segment = logical_elem_offset % NUM_ELEMS_PER_SEG; + // TMA stores the round data in a swizzled layout. swizzle_mask describes how the + // logical element offset is mapped to the actual shared-memory offset. + uint32_t swizzle_mask = Base::sw_msk(logical_elem_offset); + uint32_t chunk_swizzle_mask = swizzle_mask & ELEMS_PER_THREAD_PER_ROUND_MASK; + uint32_t smem_read_offset = logical_elem_offset ^ (swizzle_mask & ~ELEMS_PER_THREAD_PER_ROUND_MASK); + + if (main_round_idx + TMA_PREFETCH_DEPTH < num_main_rounds) { + if (cute::elect_one_sync()) { + issue_tma_loads_for_round.template operator()(main_round_idx + TMA_PREFETCH_DEPTH); + } + } + + bool is_warp_active = true; + if constexpr (HAS_PARTIAL_ROUNDS) { + is_warp_active = (num_init_rounds + main_round_idx) * NUM_SEGS_PER_ROUND + warp_idx < NUM_TAIL_SEGS + num_perm_segs; + } + + const ValueT *buf = smem.tma_load_buf[tma_buf_idx]; + // One bit per element of this thread's slice, in element order. + // Hits are rare, so appending through the set bits of this mask is faster than testing + storing every element again. + static_assert(NUM_ELEMS_PER_THREAD_PER_ROUND <= 32); // hit_mask is a uint32_t + uint32_t hit_mask = 0; + if (is_warp_active) { + smem.tma_load_full_bar[tma_buf_idx].wait(tma_buf_phase); + ValueT values[NUM_ELEMS_PER_THREAD_PER_ROUND]; + Base::template load_swizzled_slice(values, buf, smem_read_offset, chunk_swizzle_mask); + + float threshold = __uint_as_float(threshold_bits); + CUTE_UNROLL + for (uint32_t j = 0; j < NUM_ELEMS_PER_THREAD_PER_ROUND; ++j) { + asm volatile ( + "{\n" + ".reg .pred p;\n" + "setp.gtu.f32 p, %1, %2;\n" // .gtu: NaN always counts as a hit, so every NaN reaches the incoming buffer and the census reports it + "@p or.b32 %0, %0, %3;\n" + "}\n" + : "+r"(hit_mask) + : "f"((float)values[j]), "f"(threshold), "r"(1u << j)); + } + } + uint32_t num_new_incomers = __popc(hit_mask); + + uint32_t warp_total_hits = __reduce_add_sync(0xFFFFFFFF, num_new_incomers); + if (lane_idx == 0) { + smem.warp_cnt[warp_idx] = warp_total_hits; + } + __syncthreads(); + + uint32_t seg_elem_base = current_permuted_segment * NUM_ELEMS_PER_SEG + offset_in_segment; + // Element e of this thread's slice lives at smem_read_offset + (e ^ chunk_swizzle_mask) + uint32_t elem_addr_base = cute::cast_smem_ptr_to_uint(buf + smem_read_offset); + + // Start the first hit's smem load before the count exchange below, so that its latency + // (and the barrier wait) overlaps with them instead of delaying the first store. + uint32_t first_hit_e = hit_mask != 0 ? __ffs(hit_mask) - 1u : 0u; + uint32_t first_hit_val = 0; + if (is_warp_active && warp_total_hits != 0) { + asm volatile ("ld.shared.u32 %0, [%1];" : "=r"(first_hit_val) : "r"(elem_addr_base + ((first_hit_e ^ chunk_swizzle_mask) << 2))); + } + + static_assert(NUM_WARPS <= 32); + uint32_t stored_warp_hits = lane_idx < NUM_WARPS ? smem.warp_cnt[lane_idx] : 0u; + uint32_t num_total_hits_in_this_round = __reduce_add_sync(0xFFFFFFFF, stored_warp_hits); + + uint32_t dst_slot = + num_incomers + + __reduce_add_sync(0xFFFFFFFF, lane_idx < warp_idx ? stored_warp_hits : 0u) + + warp_level_exclusive_prefix_sum(num_new_incomers, lane_idx); + + if (is_warp_active && warp_total_hits != 0) { + uint32_t dst_ptr = cute::cast_smem_ptr_to_uint(smem.incoming_topk_pairs) + dst_slot * (uint32_t)sizeof(uint64_t); + if (hit_mask != 0) { + asm volatile ("st.shared.v2.u32 [%0], {%1, %2};" :: "r"(dst_ptr), "r"(seg_elem_base + first_hit_e), "r"(first_hit_val) : "memory"); + dst_ptr += (uint32_t)sizeof(uint64_t); + } + uint32_t mask = hit_mask & (hit_mask - 1u); + while (mask != 0) { + uint32_t e = __ffs(mask) - 1u; + mask &= mask - 1u; + uint32_t val_word; + asm volatile ("ld.shared.u32 %0, [%1];" : "=r"(val_word) : "r"(elem_addr_base + ((e ^ chunk_swizzle_mask) << 2))); + asm volatile ("st.shared.v2.u32 [%0], {%1, %2};" :: "r"(dst_ptr), "r"(seg_elem_base + e), "r"(val_word) : "memory"); + dst_ptr += (uint32_t)sizeof(uint64_t); + } + } + Base::advance_perm_state(current_permuted_segment, permuted_segment_stride_per_round, perm_len); + + num_incomers += num_total_hits_in_this_round; + __syncthreads(); + + if (++tma_buf_idx == NUM_TMA_LOAD_BUFS) { + tma_buf_idx = 0; + tma_buf_phase ^= 1u; + } + + if (num_incomers >= RECONSTRUCT_THRESHOLD) { + threshold_bits = reconstruct(num_incomers); + num_incomers = 0; + } + } + + if (num_incomers > 0) { + reconstruct(num_incomers); + num_incomers = 0; + } + + have_nan = __syncthreads_or(have_nan) != 0; + if (have_nan) { + Base::take_action_when_have_nan(args, batch_idx); + return; + } + + Base::template stage_output_and_epilogue( + smem, *reinterpret_cast(smem.incoming_topk_pairs), args, + batch_idx, end_vocab_idx, survivor_buf_idx, warp_idx, lane_idx); + + } + +}; + + +template +__launch_bounds__(Kernel::NUM_THREADS, Kernel::TARGET_OCCUPANCY, 1) +__global__ void topk_kernel(__grid_constant__ const TopkSelectArgs args, __grid_constant__ const typename Kernel::TmaParams tma_params) { + Kernel::topk_select_kernel_devfunc(args, tma_params); +} + +template +void run_topk_select_kernel(const TopkSelectArgs &args) { + KU_ASSERT(args.sorted_value == Config::sorted_value, "Dispatch failure"); + KU_ASSERT(args.sorted_index == Config::sorted_index, "Dispatch failure"); + KU_ASSERT(args.return_value == Config::return_value, "Dispatch failure"); + static_assert(cute::is_same_v); + KU_ASSERT(args.vocab_size < MAX_VOCAB_SIZE, "`vocab_size` is too big"); + + using Kernel = TopkSelectKernelFP32; + KU_ASSERT(args.topk <= Kernel::MAX_TOPK, "topk is too large. Maximum allowed: %d\n", Kernel::MAX_TOPK); + static_assert(INPUT_STRIDE_ALIGNMENT_REQUIREMENT % 16 == 0); + + auto kernel = topk_kernel; + constexpr size_t smem_size = sizeof(typename Kernel::SharedMemoryPlanFP32); + KU_ASSERT(smem_size * Kernel::TARGET_OCCUPANCY <= args.shared_memory_size_per_sm); + KU_CUDA_CHECK(cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, smem_size)); + + KU_ASSERT(args.stride_input_batch % 4 == 0, "stride_input_batch must be 16B-aligned"); + typename Kernel::TmaParams tma_params = {Kernel::make_topk_tensor_map(args)}; + + ku::launch_kernel(ku::KernelLaunchConfig { + dim3(args.batch_size), + dim3(Kernel::NUM_THREADS), + smem_size, + args.stream + }, kernel, args, tma_params); +} + +} // topk_select_fp32 diff --git a/python/sglang/kernels/jit/csrc/deepselect/vendor/structs.h b/python/sglang/kernels/jit/csrc/deepselect/vendor/structs.h new file mode 100644 index 0000000000..934e5f14ad --- /dev/null +++ b/python/sglang/kernels/jit/csrc/deepselect/vendor/structs.h @@ -0,0 +1,39 @@ +#pragma once + +#include +#include + +static constexpr uint32_t INPUT_STRIDE_ALIGNMENT_REQUIREMENT = 1024; // In number of bytes +static constexpr uint32_t OUTPUT_STRIDE_ALIGNMENT_REQUIREMENT = 32; // In number of bytes + +static constexpr uint32_t MAX_INT_ADDITION_RANGE_BY_FP32_SIMULATION = 1u << 23; +static constexpr uint32_t MAX_VOCAB_SIZE = 1u << 23; +static_assert(MAX_VOCAB_SIZE <= MAX_INT_ADDITION_RANGE_BY_FP32_SIMULATION); + +struct TopkSelectArgs { + uint32_t batch_size; + uint32_t vocab_size; + uint32_t topk; + + void* input; + void* output_value; + void* output_index; + int* begin_ptr; + int* end_ptr; + int* output_idx_offset; + + // All strides are in number of elements, not bytes + uint64_t stride_input_batch; + uint64_t stride_output_value_batch; + uint64_t stride_output_index_batch; + + bool sorted_value; + bool sorted_index; + bool return_value; + int idx_oob_fill_value; + float value_oob_fill_value; + bool abort_when_nan_found; + + uint64_t shared_memory_size_per_sm; + cudaStream_t stream; +}; diff --git a/python/sglang/kernels/jit/include/sgl_kernel/runtime.cuh b/python/sglang/kernels/jit/include/sgl_kernel/runtime.cuh index f8affdebef..208292ec7c 100644 --- a/python/sglang/kernels/jit/include/sgl_kernel/runtime.cuh +++ b/python/sglang/kernels/jit/include/sgl_kernel/runtime.cuh @@ -8,6 +8,8 @@ #include +#include + #include #include #include @@ -67,8 +69,13 @@ inline void* get_device_accessible_ptr(const tvm::ffi::TensorView& tensor) { return device_ptr; } -namespace details { - +/** + * \brief Memo for a per-device driver query, keyed by device ordinal. + * + * \tparam T Cached value type. + * \tparam kDefault The "not queried yet" sentinel. A query that can legally + * return it would be re-run on every call, so pick one it cannot. + */ template struct DeviceCacheMap { public: @@ -99,8 +106,6 @@ struct DeviceCacheMap { T m_data[kNumStaticMaxDevice]; }; -} // namespace details - // Return the maximum number of active blocks per SM for the given kernel template inline auto get_blocks_per_sm(T&& kernel, int32_t block_dim, std::size_t dynamic_smem = 0) -> uint32_t { @@ -111,7 +116,7 @@ inline auto get_blocks_per_sm(T&& kernel, int32_t block_dim, std::size_t dynamic // Return the number of SMs for the given device inline auto get_sm_count(int device_id, bool use_cache = true) -> uint32_t { - static details::DeviceCacheMap sm_count_cache; + static DeviceCacheMap sm_count_cache; return sm_count_cache.get_cached(device_id, use_cache, [](int32_t device_id) { int sm_count; CHECK_CUDA(cudaDeviceGetAttribute(&sm_count, cudaDevAttrMultiProcessorCount, device_id)); @@ -121,7 +126,7 @@ inline auto get_sm_count(int device_id, bool use_cache = true) -> uint32_t { // Return the Major compute capability for the given device inline auto get_cc_major(int device_id, bool use_cache = true) -> int { - static details::DeviceCacheMap cc_major_cache; + static DeviceCacheMap cc_major_cache; return cc_major_cache.get_cached(device_id, use_cache, [](int32_t device_id) { int cc_major; CHECK_CUDA(cudaDeviceGetAttribute(&cc_major, cudaDevAttrComputeCapabilityMajor, device_id)); @@ -131,7 +136,7 @@ inline auto get_cc_major(int device_id, bool use_cache = true) -> int { // Return the Minor compute capability for the given device inline auto get_cc_minor(int device_id, bool use_cache = true) -> int { - static details::DeviceCacheMap cc_minor_cache; + static DeviceCacheMap cc_minor_cache; return cc_minor_cache.get_cached(device_id, use_cache, [](int32_t device_id) { int cc_minor; CHECK_CUDA(cudaDeviceGetAttribute(&cc_minor, cudaDevAttrComputeCapabilityMinor, device_id)); @@ -159,6 +164,26 @@ inline auto get_available_dynamic_smem_per_block(T&& kernel, int num_blocks, int return smem_size; } +/** + * \brief Shared memory one block may use on the given device, in bytes. + * + * \param device_id CUDA device ordinal. + * \param opt_in True for the larger limit a kernel reaches by setting + * `cudaFuncAttributeMaxDynamicSharedMemorySize`, false for the default + * cap that needs no opt-in. + * \param use_cache False re-queries the driver instead of reading the memo. + */ +inline auto get_max_smem_per_block(int device_id, bool opt_in = true, bool use_cache = true) -> std::size_t { + /// NOTE: we all know this is no larger than 1MB for all GPU, so use uint32_t + static DeviceCacheMap smem_cache[2]; + const auto attr = opt_in ? cudaDevAttrMaxSharedMemoryPerBlockOptin : cudaDevAttrMaxSharedMemoryPerBlock; + return smem_cache[opt_in].get_cached(device_id, use_cache, [attr](int32_t device_id) { + int value = 0; + CHECK_CUDA(cudaDeviceGetAttribute(&value, attr, device_id)); + return value; + }); +} + } // namespace host::runtime } // namespace sglang diff --git a/python/sglang/kernels/jit/utils/__init__.py b/python/sglang/kernels/jit/utils/__init__.py index 93babedb0c..0e52be9226 100644 --- a/python/sglang/kernels/jit/utils/__init__.py +++ b/python/sglang/kernels/jit/utils/__init__.py @@ -7,6 +7,7 @@ from sglang.kernels.jit.utils.arch import ( override_jit_cuda_arch, ) from sglang.kernels.jit.utils.common import ( + aligned_new_empty, cache_once, empty_sentinel, get_ci_test_range, @@ -23,6 +24,7 @@ from sglang.kernels.jit.utils.compile import ( ) __all__ = [ + "aligned_new_empty", "empty_sentinel", "should_run_full_tests", "get_ci_test_range", diff --git a/python/sglang/kernels/jit/utils/common.py b/python/sglang/kernels/jit/utils/common.py index 9d7f09f08f..65582cd7ef 100644 --- a/python/sglang/kernels/jit/utils/common.py +++ b/python/sglang/kernels/jit/utils/common.py @@ -3,7 +3,7 @@ from __future__ import annotations import functools -from typing import Any, Callable, Dict, List, TypeVar +from typing import Any, Callable, Dict, List, Sequence, TypeVar import torch @@ -86,3 +86,28 @@ def lazy_register_class(name: str, init_fn: Callable[[], None]) -> Callable[[T], return cls return decorator + + +def aligned_new_empty( + shape: Sequence[int], + dtype: torch.dtype, + device: torch.device, + *, + alignment: int, +) -> torch.Tensor: + """An uninitialized tensor whose row stride is a multiple of `alignment` bytes. + + The last dimension is padded up and then sliced back off, so the result has + the requested shape and a non-contiguous stride whenever padding was needed. + Callers are kernels that state an alignment contract on a row address, such + as a TMA descriptor or a vectorized load. + """ + assert len(shape) > 1 + assert alignment % dtype.itemsize == 0, ( + "alignment must be a whole number of elements" + ) + step = alignment // dtype.itemsize + last_dim = shape[-1] + padded_last_dim = (last_dim + step - 1) // step * step + aligned_shape = (*shape[:-1], padded_last_dim) + return torch.empty(aligned_shape, dtype=dtype, device=device)[..., :last_dim] diff --git a/python/sglang/kernels/ops/deep_select.py b/python/sglang/kernels/ops/deep_select.py new file mode 100644 index 0000000000..abbc48082f --- /dev/null +++ b/python/sglang/kernels/ops/deep_select.py @@ -0,0 +1,320 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Dict, NamedTuple, Optional, Tuple + +import torch + +from sglang.kernels.jit.utils import ( + KERNEL_PATH, + aligned_new_empty, + cache_once, + load_jit, + make_cpp_args, +) + +if TYPE_CHECKING: + from tvm_ffi.module import Module + +# Must match INPUT_/OUTPUT_STRIDE_ALIGNMENT_REQUIREMENT in vendor/structs.h; the +# kernel's TMA descriptor depends on them and `entry.cuh` rejects a row that +# misses one. +_INPUT_STRIDE_ALIGNMENT_BYTES = 1024 +_OUTPUT_STRIDE_ALIGNMENT_BYTES = 32 +_CLUSTER_MAX_TOPK = 1024 +_CLUSTER_MAX_BATCH_SIZE = 6 +_SUPPORTED_CAPABILITIES = ((9, 0), (10, 0), (10, 3)) + + +class _Config(NamedTuple): + max_topk: int + num_threads: int + occupancy: int + elements_per_round: int + reconstruct_threshold: int + tma_buffer_depth: int + cluster_size: int = 1 + + def to_cpp_class_name( + self, + value_dtype: torch.dtype, + index_dtype: torch.dtype, + sorted_value: bool, + sorted_index: bool, + return_value: bool, + ) -> str: + template_args = make_cpp_args( + value_dtype, + index_dtype, + sorted_value, + sorted_index, + return_value, + self.max_topk, + self.num_threads, + self.occupancy, + self.elements_per_round, + self.reconstruct_threshold, + self.tma_buffer_depth, + 512, # elements_per_segment, fixed upstream + self.cluster_size, + ) + return f"TopkSelectConfig<{template_args}>" + + +# Per (value dtype, top-k bucket): the tuning for a single-wave batch and, where +# the bucket splits on wave count, the tuning for anything larger. A single +# entry means no split, and then only one kernel is instantiated. +_NORMAL_CONFIGS: Dict[Tuple[torch.dtype, int], Tuple[_Config, ...]] = { + (torch.bfloat16, 512): ( + _Config(512, 512, 1, 8192, 4096, 5), + _Config(512, 256, 2, 4096, 4096, 4), + ), + (torch.bfloat16, 1024): ( + _Config(1024, 512, 1, 8192, 4096, 5), + _Config(1024, 256, 2, 4096, 4096, 3), + ), + (torch.bfloat16, 4096): (_Config(4096, 512, 1, 8192, 4096, 3),), + (torch.float32, 512): (_Config(512, 512, 1, 8192, 4096, 3),), + (torch.float32, 1024): (_Config(1024, 512, 1, 8192, 4096, 3),), + (torch.float32, 4096): (_Config(4096, 256, 1, 4096, 4096, 3),), +} + + +@cache_once +def _cluster_tuning(device_capability: Tuple[int, int]) -> Tuple[int, int]: + # Return (cluster_size, min_vocab_size) + major = device_capability[0] + return (8, 128 * 1024) if major == 9 else (16, 512 * 1024) + + +@cache_once +def _jit_deep_select_module( + value_dtype: torch.dtype, + index_dtype: torch.dtype, + sorted_value: bool, + sorted_index: bool, + return_value: bool, + max_topk: int, + cluster_size: int, +) -> Module: + assert value_dtype in (torch.bfloat16, torch.float32) + assert index_dtype in (torch.int32, torch.int64) + if sorted_value: + assert value_dtype == torch.float32 + assert return_value and not sorted_index + if cluster_size == 1: + configs = _NORMAL_CONFIGS[(value_dtype, max_topk)] + host_dispatch = "TopkNormal" + else: + # Upstream tunes the cluster kernel for bf16 at max_topk 1024 alone, so + # the 512 and 1024 buckets share this module. + assert value_dtype == torch.bfloat16 + assert max_topk == _CLUSTER_MAX_TOPK + configs = [_Config(max_topk, 256, 1, 4096, 4096, 16, cluster_size)] + host_dispatch = "TopkCluster" + classes = [ + config.to_cpp_class_name( + value_dtype=value_dtype, + index_dtype=index_dtype, + sorted_value=sorted_value, + sorted_index=sorted_index, + return_value=return_value, + ) + for config in configs + ] + classes = make_cpp_args(*classes) + root = (KERNEL_PATH / "csrc" / "deepselect" / "vendor").resolve() + return load_jit( + "deep_select_topk", + # cache only distinct key for better readability + *make_cpp_args( + value_dtype, + index_dtype, + sorted_value, + sorted_index, + return_value, + max_topk, + cluster_size, + ), + cuda_files=["deepselect/entry.cuh"], + cuda_wrappers=[("topk", f"deepselect::{host_dispatch}<{classes}>::topk")], + extra_include_paths=[ + str(root), + str(root / "3rdparty" / "kerutils" / "include"), + ], + extra_dependencies=["cutlass"], + extra_cuda_cflags=[ + "--expt-extended-lambda", + "--use_fast_math", + "--ftz=false", + "-Xptxas=--register-usage-level=10", + ], + ) + + +def _get_max_topk_bucket(topk: int, use_cluster: bool) -> int: + if use_cluster: + return _CLUSTER_MAX_TOPK + return 512 if topk <= 512 else 1024 if topk <= 1024 else 4096 + + +def get_input_stride_alignment_bytes() -> int: + return _INPUT_STRIDE_ALIGNMENT_BYTES + + +def get_stride_requirement() -> Tuple[int, int]: + """Return the input and output row-stride requirements in bytes.""" + return _INPUT_STRIDE_ALIGNMENT_BYTES, _OUTPUT_STRIDE_ALIGNMENT_BYTES + + +def get_deepselect_supported_architectures() -> Tuple[int, ...]: + """Return the CUDA compute capabilities supported by the JIT kernel.""" + return tuple(major * 10 + minor for major, minor in _SUPPORTED_CAPABILITIES) + + +def is_deepselect_supported(device=None) -> bool: + """Return whether DeepSelect JIT supports a CUDA device.""" + if torch.version.cuda is None or not torch.cuda.is_available(): + return False + try: + normalized_device = ( + torch.device("cuda", device) + if isinstance(device, int) + else ( + torch.device("cuda", torch.cuda.current_device()) + if device is None + else torch.device(device) + ) + ) + if normalized_device.type != "cuda": + return False + return ( + torch.cuda.get_device_capability(normalized_device) + in _SUPPORTED_CAPABILITIES + ) + except (RuntimeError, TypeError, ValueError): + return False + + +def _needs_output_staging(output: torch.Tensor, topk: int) -> bool: + return ( + output.data_ptr() % _OUTPUT_STRIDE_ALIGNMENT_BYTES != 0 + or topk * output.element_size() % _OUTPUT_STRIDE_ALIGNMENT_BYTES != 0 + ) + + +def topk( + input: torch.Tensor, + topk: int, + sorted: bool = False, + begin: Optional[torch.Tensor] = None, + end: Optional[torch.Tensor] = None, + indices_type: torch.dtype = torch.int64, + sorted_index: bool = False, + hint: Optional[torch.Tensor] = None, + output_idx: Optional[torch.Tensor] = None, + output_idx_offset: Optional[torch.Tensor] = None, + idx_oob_fill_value: int = 2147483647, + value_oob_fill_value: float = float("-inf"), + return_value: bool = True, + abort_when_nan_found: bool = True, +) -> Tuple[Optional[torch.Tensor], torch.Tensor]: + """Select the largest ``topk`` values of every row of ``input``. + + This follows the public DeepSelect interface. ``end`` is the per-row + exclusive valid length; ``begin`` and ``hint`` are reserved but unsupported. + + Returns ``(values, indices)``; ``values`` is None when ``return_value`` is + False, which skips writing them and is about 10% faster. + """ + if hint is not None: + raise ValueError("hint is not supported currently") + if input.dtype not in (torch.bfloat16, torch.float32): + raise RuntimeError("input dtype must be bfloat16 or float32") + if indices_type not in (torch.int32, torch.int64): + raise RuntimeError("indices_type must be int32 or int64") + if output_idx is not None and output_idx.dtype != indices_type: + raise ValueError("output_idx dtype must match indices_type") + if input.device.type != "cuda": + raise RuntimeError("input must be a CUDA tensor") + if input.ndim != 2: + raise RuntimeError("input must be a 2D tensor") + if not 0 < topk <= 4096: + raise RuntimeError(f"topk must be in [1, 4096], got {topk}") + if sorted and not return_value: + raise RuntimeError("return_value must be enabled when sorted is True") + if sorted and sorted_index: + raise RuntimeError("sorted and sorted_index cannot both be True") + if sorted and input.dtype is torch.bfloat16: + raise RuntimeError("sorted is only supported for float32 input") + + rows, vocab_size = input.shape + device_capability = torch.cuda.get_device_capability(input.device) + if device_capability not in _SUPPORTED_CAPABILITIES: + major, minor = device_capability + raise RuntimeError(f"DeepSelect does not support SM{major}{minor}") + cluster_size, cluster_min_vocab_size = _cluster_tuning(device_capability) + use_cluster = ( + input.dtype is torch.bfloat16 + and rows <= _CLUSTER_MAX_BATCH_SIZE + and vocab_size >= cluster_min_vocab_size + and topk <= _CLUSTER_MAX_TOPK + ) + values = None + if return_value: + values = aligned_new_empty( + (rows, topk), + input.dtype, + input.device, + alignment=_OUTPUT_STRIDE_ALIGNMENT_BYTES, + ) + if output_idx is None: + output_idx = aligned_new_empty( + (rows, topk), + indices_type, + input.device, + alignment=_OUTPUT_STRIDE_ALIGNMENT_BYTES, + ) + kernel_output_idx = output_idx + else: + kernel_output_idx = ( + aligned_new_empty( + (rows, topk), + indices_type, + input.device, + alignment=_OUTPUT_STRIDE_ALIGNMENT_BYTES, + ) + if _needs_output_staging(output_idx, topk) + else output_idx + ) + module = _jit_deep_select_module( + input.dtype, + indices_type, + sorted, + sorted_index, + return_value, + _get_max_topk_bucket(topk, use_cluster), + cluster_size if use_cluster else 1, + ) + input_storage_bytes = ( + input.untyped_storage().nbytes() - input.storage_offset() * input.element_size() + ) + module.topk( + input, + values, + output_idx, + kernel_output_idx, + begin, + end, + output_idx_offset, + topk, + input_storage_bytes, + idx_oob_fill_value, + value_oob_fill_value, + abort_when_nan_found, + ) + if kernel_output_idx is not output_idx: + output_idx.copy_(kernel_output_idx) + return values, output_idx + + +deepselect_topk = topk diff --git a/test/registered/kernels/ops/attention/test_deep_select.py b/test/registered/kernels/ops/attention/test_deep_select.py new file mode 100644 index 0000000000..450b2f7b0e --- /dev/null +++ b/test/registered/kernels/ops/attention/test_deep_select.py @@ -0,0 +1,95 @@ +"""Regression tests for the DeepSelect JIT wrapper and vendored kernels.""" + +from __future__ import annotations + +import inspect + +import pytest +import torch + +from sglang.kernels.ops.deep_select import is_deepselect_supported, topk +from sglang.test.ci.ci_register import register_cuda_ci + +register_cuda_ci(est_time=240, stage="base-b-kernel-unit", runner_config="1-gpu-large") + +pytestmark = pytest.mark.skipif( + not is_deepselect_supported(), reason="requires CUDA SM90, SM100, or SM103" +) + + +def _aligned_input( + rows: int, width: int, dtype: torch.dtype, device: torch.device | str = "cuda" +) -> torch.Tensor: + alignment = 1024 // dtype.itemsize + padded_width = (width + alignment - 1) // alignment * alignment + return torch.empty((rows, padded_width), dtype=dtype, device=device)[:, :width] + + +def test_deepselect_topk_matches_official_signature(): + assert list(inspect.signature(topk).parameters) == [ + "input", + "topk", + "sorted", + "begin", + "end", + "indices_type", + "sorted_index", + "hint", + "output_idx", + "output_idx_offset", + "idx_oob_fill_value", + "value_oob_fill_value", + "return_value", + "abort_when_nan_found", + ] + + +def test_out_of_range_oob_fill_is_rejected(): + input = _aligned_input(1, 1024, torch.float32) + with pytest.raises(RuntimeError, match="idx_oob_fill_value must fit in int32"): + topk(input, 8, idx_oob_fill_value=1 << 40) + + +def test_misaligned_caller_output_is_staged_without_overwrite(): + input = _aligned_input(1, 1024, torch.float32) + input.copy_(torch.arange(1024, device=input.device)) + backing = torch.full((528,), -77, dtype=torch.int64, device=input.device) + output = backing.as_strided((1, 513), (520, 1), 1) + + values, indices = topk(input, 513, sorted=True, output_idx=output) + + assert values is not None + assert indices is output + expected = torch.topk(input, 513) + torch.testing.assert_close(indices, expected.indices, rtol=0, atol=0) + torch.testing.assert_close(values, expected.values, rtol=0, atol=0) + assert backing[0] == -77 + assert torch.all(backing[514:] == -77) + + +@pytest.mark.parametrize( + "layout,error", + [ + ("misaligned", "input address must be aligned"), + ("missing-tail-padding", "input storage must include"), + ], +) +def test_input_storage_validation(layout, error): + dtype = torch.bfloat16 + width = 1024 // dtype.itemsize + 1 + stride = 2 * (1024 // dtype.itemsize) + if layout == "misaligned": + backing = torch.empty(stride + 1, dtype=dtype, device="cuda") + input = backing.as_strided((1, width), (stride, 1), 1) + else: + input = torch.empty_strided((1, width), (stride, 1), dtype=dtype, device="cuda") + + with pytest.raises(RuntimeError, match=error): + topk(input, 8) + + +def test_begin_is_rejected_by_cpp(): + input = _aligned_input(1, 1024, torch.float32) + begin = torch.zeros(1, dtype=torch.int32, device=input.device) + with pytest.raises(RuntimeError, match="`begin` is not supported"): + topk(input, 8, begin=begin)