[DeepSeek V4.1] Add DeepSelect JIT kernel. (#40556)

Co-authored-by: DarkSharpness <2040703891@qq.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
yuyu5333
2026-09-26 17:36:48 +08:00
committed by GitHub
co-authored by DarkSharpness Claude Opus 5
parent a5c8afab48
commit 7bc988446d
27 changed files with 5016 additions and 8 deletions
@@ -0,0 +1,59 @@
# DeepSelect top-k
`vendor/` is [deepseek-ai/DeepSelect](https://github.com/deepseek-ai/DeepSelect)
(commit `382d62a`, plus the SM90 cluster variant from upstream PR #14), stripped
to the kernel implementation. MIT licensed; see `vendor/LICENSE`.
`entry.cuh` is the SGLang side: tensor validation, the `TopkSelectArgs` block,
and the two host entry points the JIT exports. `python/sglang/kernels/ops/deep_select.py`
is the wrapper that compiles and calls them.
## What is vendored
| Path under `vendor/` | What it is |
| --- | --- |
| `structs.h` | `TopkSelectArgs` -- the launcher's plain-C argument block, plus the stride-alignment constants |
| `cuda_kernels/config.h` | `TopkSelectConfig<...>` -- every compile-time knob of a kernel instance |
| `cuda_kernels/{utils,bit_utils}.cuh` | small device helpers |
| `cuda_kernels/common_parts.cuh` | the shared kernel body (TMA pipeline, radix pass, epilogue) |
| `cuda_kernels/v3/topk_select.cuh` | bf16, one CTA per row |
| `cuda_kernels/v3_cluster/topk_select.cuh` | bf16, one cluster per row (8 CTAs on SM90, 16 on SM100/SM103) |
| `cuda_kernels/v3_fp32/topk_select.cuh` | fp32 |
| `3rdparty/kerutils/` | upstream's header-only helper library, subset |
Each `topk_select.cuh` defines one kernel plus its launcher,
`run_topk_select_kernel<Config>(const TopkSelectArgs&)`, in its own namespace
(`topk_select_bf16_normal`, `topk_select_bf16_cluster`, `topk_select_fp32`).
Dropped from the vendored tree, because a JIT build does not need them:
- `cuda_kernels/*/instantiations/**` -- 86 explicit instantiation `.cu` files.
They exist so an AOT build can compile every configuration into one shared
object. `entry.cuh` instantiates the one or two a module can reach.
- `api.cpp` -- the torch/pybind entry point and the runtime dispatch that
picked a configuration from `(dtype, topk, vocab_size, batch_size, arch)`.
That decision is now split between the Python module factory and `entry.cuh`.
- `dispatch_utils.h` -- `BOOL_SWITCH` / `INTEGER_TYPE_SWITCH`, which include
`torch/extension.h` and exist only to expand that dispatch.
- `cuda_kernels/*/topk_select.h` -- forward declarations of
`run_topk_select_kernel`, needed only to split the instantiations into
separate translation units.
## Build inputs
Include paths: `vendor/` (the `#include "cuda_kernels/..."` lines resolve
against it) and `vendor/3rdparty/kerutils/include`. CUTLASS supplies
`cute/arch/copy_sm90_tma.hpp` and `cutlass/arch/barrier.h`, and is a registered
JIT dependency (`extra_dependencies=["cutlass"]`).
Beyond the JIT defaults the kernels need `--expt-extended-lambda` and
`--ftz=false`; the latter is load-bearing and must follow `--use_fast_math`,
because the radix pass simulates integer counters with fp32 addition and needs
subnormals.
## Re-syncing with upstream
Copy the files listed above into `vendor/`. They are unmodified apart from
trailing whitespace and the removed `#include "topk_select.h"` line at the top
of each `topk_select.cuh`. `vendor/.clang-format` disables formatting so a
re-sync stays a plain copy.
@@ -0,0 +1,294 @@
#pragma once
// The vendored DeepSelect sources declare everything at global scope, so they
// are included before `namespace sglang` is opened.
#include <sgl_kernel/tensor.h>
#include <sgl_kernel/utils.h>
#include <sgl_kernel/runtime.cuh>
#include <sgl_kernel/utils.cuh>
#include <tvm/ffi/container/tensor.h>
#include <tvm/ffi/optional.h>
#include "vendor/cuda_kernels/v3/topk_select.cuh"
#include "vendor/cuda_kernels/v3_cluster/topk_select.cuh"
#include "vendor/cuda_kernels/v3_fp32/topk_select.cuh"
#include <cstdint>
#include <limits>
#include <type_traits>
namespace sglang {
/**
* \brief Host entry points for the vendored DeepSelect top-k kernels.
*
* Upstream resolves a `TopkSelectConfig` from the call at runtime, expanding
* every boolean and index-type axis through nested `BOOL_SWITCH` macros. Here
* those axes -- value dtype, index dtype, `sorted_value`, `sorted_index`,
* `return_value`, the top-k bucket and the cluster size -- are template
* arguments picked by the Python module factory, which leaves exactly one
* decision for C++: which of a bucket's two tunings the launch shape wants.
*/
namespace deepselect {
namespace details {
/// The two families of one-CTA-per-row kernels live in separate namespaces.
template <typename Config>
inline auto run_normal(const TopkSelectArgs& args) -> void {
if constexpr (std::is_same_v<typename Config::ValueT, float>) {
topk_select_fp32::run_topk_select_kernel<Config>(args);
} else {
topk_select_bf16_normal::run_topk_select_kernel<Config>(args);
}
}
/// Validated launch arguments plus the device they were validated against.
struct LaunchPlan {
TopkSelectArgs args;
DLDevice device;
};
/**
* \brief Validate the tensors and build the argument block for one launch.
*
* \tparam Config The specialisation the caller is about to run. Its
* `sorted_value` / `sorted_index` / `return_value` become the runtime
* flags, which the kernel launcher then asserts back against them.
*/
template <typename Config>
auto make_plan(
tvm::ffi::TensorView input,
tvm::ffi::Optional<tvm::ffi::TensorView> output_value,
tvm::ffi::TensorView output_index,
tvm::ffi::TensorView kernel_output_index,
tvm::ffi::Optional<tvm::ffi::TensorView> begin,
tvm::ffi::Optional<tvm::ffi::TensorView> end,
tvm::ffi::Optional<tvm::ffi::TensorView> output_index_offset,
int64_t topk,
int64_t input_storage_bytes,
int64_t idx_oob_fill_value,
double value_oob_fill_value,
bool abort_when_nan_found) -> LaunchPlan {
using namespace host;
using ValueT = typename Config::ValueT;
using OutIdxT = typename Config::OutIdxT;
auto batch = SymbolicSize{"batch_size"};
auto vocab = SymbolicSize{"vocab_size"};
auto input_stride = SymbolicSize{"input_row_stride"};
auto index_stride = SymbolicSize{"index_row_stride"};
auto kernel_index_stride = SymbolicSize{"kernel_index_row_stride"};
auto value_stride = SymbolicSize{"value_row_stride"};
auto device = SymbolicDevice{};
CHECK_HOST(topk > 0 && topk <= static_cast<int64_t>(Config::max_topk))
<< "topk must be in (0, " << Config::max_topk << "] for this module, got " << topk;
CHECK_HOST(!begin.has_value()) << "`begin` is not supported currently";
CHECK_HOST(
idx_oob_fill_value >= std::numeric_limits<int32_t>::min() &&
idx_oob_fill_value <= std::numeric_limits<int32_t>::max())
<< "idx_oob_fill_value must fit in int32";
CHECK_HOST(input_storage_bytes >= 0) << "input_storage_bytes must be non-negative";
TensorMatcher({batch, vocab})
.with_strides({input_stride, 1})
.with_device<kDLCUDA>(device)
.with_dtype<ValueT>()
.verify(input);
TensorMatcher({batch, topk})
.with_strides({index_stride, 1})
.with_device<kDLCUDA>(device)
.with_dtype<OutIdxT>()
.verify(output_index);
TensorMatcher({batch, topk})
.with_strides({kernel_index_stride, 1})
.with_device<kDLCUDA>(device)
.with_dtype<OutIdxT>()
.ensure_alignment(OUTPUT_STRIDE_ALIGNMENT_REQUIREMENT)
.verify(kernel_output_index);
void* output_value_ptr = nullptr;
uint64_t stride_output_value = 0;
if constexpr (Config::return_value) {
CHECK_HOST(output_value.has_value()) << "output_value is required when return_value is enabled";
TensorMatcher({batch, topk})
.with_strides({value_stride, 1})
.with_device<kDLCUDA>(device)
.with_dtype<ValueT>()
.ensure_alignment(OUTPUT_STRIDE_ALIGNMENT_REQUIREMENT)
.verify(output_value.value());
output_value_ptr = output_value.value().data_ptr();
stride_output_value = static_cast<uint64_t>(value_stride.unwrap());
} else {
CHECK_HOST(!output_value.has_value()) << "output_value must be absent when return_value is disabled";
}
int* end_ptr = nullptr;
if (end.has_value()) {
TensorMatcher({batch}).with_device<kDLCUDA>(device).with_dtype<int32_t>().verify(end.value());
end_ptr = static_cast<int*>(end.value().data_ptr());
}
int* index_offset_ptr = nullptr;
if (output_index_offset.has_value()) {
TensorMatcher({batch}).with_device<kDLCUDA>(device).with_dtype<int32_t>().verify(output_index_offset.value());
index_offset_ptr = static_cast<int*>(output_index_offset.value().data_ptr());
}
const auto batch_size = static_cast<uint32_t>(batch.unwrap());
const auto vocab_size = static_cast<uint32_t>(vocab.unwrap());
CHECK_HOST(vocab_size < MAX_VOCAB_SIZE) << "vocab_size must be < " << MAX_VOCAB_SIZE << ", got " << vocab_size;
const DLDevice dev = device.unwrap();
const auto input_stride_elements = input_stride.unwrap();
CHECK_HOST(input_stride_elements >= 0) << "input row stride must be non-negative";
CHECK_HOST(static_cast<uint64_t>(input_stride_elements) * sizeof(ValueT) % INPUT_STRIDE_ALIGNMENT_REQUIREMENT == 0)
<< "input row stride must be a multiple of " << INPUT_STRIDE_ALIGNMENT_REQUIREMENT << " bytes";
CHECK_HOST(static_cast<uint64_t>(index_stride.unwrap()) * sizeof(OutIdxT) % OUTPUT_STRIDE_ALIGNMENT_REQUIREMENT == 0)
<< "output_index row stride must be a multiple of " << OUTPUT_STRIDE_ALIGNMENT_REQUIREMENT << " bytes";
if (batch_size != 0 && vocab_size != 0) {
constexpr uint64_t address_alignment = SGL_CUDA_ARCH == 900 ? 16 : 32;
CHECK_HOST(reinterpret_cast<uintptr_t>(input.data_ptr()) % address_alignment == 0)
<< "input address must be aligned to " << address_alignment << " bytes";
const uint64_t row_bytes = static_cast<uint64_t>(vocab_size) * sizeof(ValueT);
const uint64_t padded_row_bytes = (row_bytes + 127) / 128 * 128;
const uint64_t required_bytes =
static_cast<uint64_t>(batch_size - 1) * input_stride_elements * sizeof(ValueT) + padded_row_bytes;
CHECK_HOST(required_bytes <= static_cast<uint64_t>(input_storage_bytes))
<< "input storage must include the final row padded to 128 bytes";
}
return LaunchPlan{
TopkSelectArgs{
batch_size,
vocab_size,
static_cast<uint32_t>(topk),
input.data_ptr(),
output_value_ptr,
kernel_output_index.data_ptr(),
nullptr, // `begin` is unimplemented upstream
end_ptr,
index_offset_ptr,
static_cast<uint64_t>(input_stride.unwrap()),
stride_output_value,
static_cast<uint64_t>(kernel_index_stride.unwrap()),
Config::sorted_value,
Config::sorted_index,
Config::return_value,
static_cast<int>(idx_oob_fill_value),
static_cast<float>(value_oob_fill_value),
abort_when_nan_found,
host::runtime::get_max_smem_per_block(dev.device_id),
host::LaunchKernel::resolve_device(dev),
},
dev,
};
}
} // namespace details
/**
* \brief Top-k through the one-CTA-per-row kernels.
*
* \tparam ConfigWave1 Tuning for a batch that fits in a single wave.
* \tparam ConfigWaves Tuning for anything larger. A bucket with no wave split
* leaves it defaulted; one kernel is then instantiated and the SM-count
* query drops out.
*/
template <typename ConfigWave1, typename ConfigWaves = ConfigWave1>
struct TopkNormal {
static_assert(ConfigWave1::cluster_size == 1 && ConfigWaves::cluster_size == 1);
static_assert(std::is_same_v<typename ConfigWave1::ValueT, typename ConfigWaves::ValueT>);
static_assert(std::is_same_v<typename ConfigWave1::OutIdxT, typename ConfigWaves::OutIdxT>);
static_assert(ConfigWave1::max_topk == ConfigWaves::max_topk);
static_assert(ConfigWave1::sorted_value == ConfigWaves::sorted_value);
static_assert(ConfigWave1::sorted_index == ConfigWaves::sorted_index);
static_assert(ConfigWave1::return_value == ConfigWaves::return_value);
static auto topk(
tvm::ffi::TensorView input,
tvm::ffi::Optional<tvm::ffi::TensorView> output_value,
tvm::ffi::TensorView output_index,
tvm::ffi::TensorView kernel_output_index,
tvm::ffi::Optional<tvm::ffi::TensorView> begin,
tvm::ffi::Optional<tvm::ffi::TensorView> end,
tvm::ffi::Optional<tvm::ffi::TensorView> output_index_offset,
int64_t topk,
int64_t input_storage_bytes,
int64_t idx_oob_fill_value,
double value_oob_fill_value,
bool abort_when_nan_found) -> void {
const auto plan = details::make_plan<ConfigWave1>(
input,
output_value,
output_index,
kernel_output_index,
begin,
end,
output_index_offset,
topk,
input_storage_bytes,
idx_oob_fill_value,
value_oob_fill_value,
abort_when_nan_found);
if (plan.args.batch_size == 0) return;
if constexpr (std::is_same_v<ConfigWave1, ConfigWaves>) {
details::run_normal<ConfigWave1>(plan.args);
} else {
const auto num_sm = host::runtime::get_sm_count(plan.device.device_id);
if (plan.args.batch_size <= num_sm) {
details::run_normal<ConfigWave1>(plan.args);
} else {
details::run_normal<ConfigWaves>(plan.args);
}
}
}
};
/**
* \brief Top-k through the cluster kernel: `Config::cluster_size` CTAs per row.
*
* Whether a launch belongs here -- small batch, long rows, bf16 -- is decided
* in Python, which also picks the cluster size: 8 CTAs on SM90, 16 on
* SM100/SM103, where 16 needs the non-portable cluster attribute.
*/
template <typename Config>
struct TopkCluster {
static_assert(Config::cluster_size > 1);
static auto topk(
tvm::ffi::TensorView input,
tvm::ffi::Optional<tvm::ffi::TensorView> output_value,
tvm::ffi::TensorView output_index,
tvm::ffi::TensorView kernel_output_index,
tvm::ffi::Optional<tvm::ffi::TensorView> begin,
tvm::ffi::Optional<tvm::ffi::TensorView> end,
tvm::ffi::Optional<tvm::ffi::TensorView> output_index_offset,
int64_t topk,
int64_t input_storage_bytes,
int64_t idx_oob_fill_value,
double value_oob_fill_value,
bool abort_when_nan_found) -> void {
const auto plan = details::make_plan<Config>(
input,
output_value,
output_index,
kernel_output_index,
begin,
end,
output_index_offset,
topk,
input_storage_bytes,
idx_oob_fill_value,
value_oob_fill_value,
abort_when_nan_found);
if (plan.args.batch_size == 0) return;
topk_select_bf16_cluster::run_topk_select_kernel<Config>(plan.args);
}
};
} // namespace deepselect
} // namespace sglang
@@ -0,0 +1,4 @@
# Vendored from deepseek-ai/DeepSelect; keep the sources byte-comparable with
# upstream so re-syncing stays a plain copy. See README.md.
DisableFormat: true
SortIncludes: Never
@@ -0,0 +1,20 @@
# Kerutils
Kerutils is a library that provides:
- Wrappers for low-level PTX instructions
- Helpers for checking tensor device / shape / stride / dtype, useful in kernel library's dispatcher
> This copy is a subset of upstream Kerutils: the CuTe UTCMMA / 2SM TMA copy wrappers,
> the GeMM helpers and the Kernel Insight (KI) library have been removed because
> DeepSelect does not use them.
## Getting Started
To use Kerutils, simply clone this repo (or register it as a submodule, if you're working in a repo), and include `<kerutils/kerutils.cuh>`. You should add `include/` into your compiler's `includePath` (for example, by adding `-Ipath/to/kerutils/include`).
## Organization
- `include/host` - Host-side helper functions
- `include/device` - Device-side PTX wrappers and helper functions, organized by GPU architectures
- `supplemental/` - Files in this directory are NOT included by `kerutils/kerutils.cuh` by default, to reduce compilation time. `supplemental/torch_tensors.h` provides various handy guard functions for examining a libTorch tensor.
@@ -0,0 +1,15 @@
#pragma once
namespace kerutils {}
#define KU_PRINTLN(fmt, ...) { cute::print(fmt, ##__VA_ARGS__); print("\n"); }
namespace ku = kerutils;
#ifdef __CUDACC__
#define KERUTILS_IS_BUILD_ON_CUDA
#endif
#ifndef KERUTILS_IS_BUILD_ON_CUDA
#error "KERUTILS_IS_BUILD_ON_CUDA must be defined. It is defined automatically when compiling with NVCC; pass `-DKERUTILS_IS_BUILD_ON_CUDA` when compiling with a host compiler."
#endif
@@ -0,0 +1,75 @@
/*
Common data types and macros that are used across the kerutils library.
*/
#pragma once
#include <cuda_bf16.h>
#include <cuda_fp8.h>
#include <cutlass/bfloat16.h>
#include <cutlass/arch/barrier.h>
#include <cute/config.hpp> // For CUTE_DEVICE
namespace kerutils {
// Cache hints
enum class CacheHint {
EVICT_FIRST,
EVICT_NORMAL,
EVICT_LAST,
EVICT_UNCHANGED,
NO_ALLOCATE
};
// Prefetch size
enum class PrefetchSize {
B64,
B128,
B256
};
using nvbf16 = __nv_bfloat16;
using nvbf16x2 = __nv_bfloat162;
using nve4m3 = __nv_fp8_e4m3;
using nve4m3x2 = __nv_fp8x2_e4m3;
using nve4m3x4 = __nv_fp8x4_e4m3;
using bf16 = cutlass::bfloat16_t;
using transac_bar_t = cutlass::arch::ClusterTransactionBarrier;
}
#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 800))
#define KERUTILS_ENABLE_SM80
#elif (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ < 800))
static_assert(false, "kerutils doesn't support SM architectures below SM80");
#endif
#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900))
#define KERUTILS_ENABLE_SM90
#endif
#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900 && __CUDA_ARCH__ < 1000))
#define KERUTILS_ENABLE_SM90A
#endif
#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000))
#define KERUTILS_ENABLE_SM100
#endif
#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000 && __CUDA_ARCH__ < 1200))
#define KERUTILS_ENABLE_SM100A
#endif
#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1030 && __CUDA_ARCH__ < 1200))
#define KERUTILS_ENABLE_SM103A
#endif
#if (defined(__CLION_IDE__) || defined(__VSCODE_IDE__))
#define KERUTILS_ENABLE_SM80
#define KERUTILS_ENABLE_SM90
#define KERUTILS_ENABLE_SM90A
#define KERUTILS_ENABLE_SM100
#define KERUTILS_ENABLE_SM100A
#define KERUTILS_ENABLE_SM103A
#endif
@@ -0,0 +1,649 @@
#pragma once
#include <cute/atom/copy_traits_sm100.hpp>
#include "kerutils/device/cuda/common.h"
namespace kerutils {
// tma gather4 (https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-cp-async-bulk-tensor)
// Please pay attention that the coordinates of TMA gather4 are int32, which may lead to overflow under some scenarios
CUTE_DEVICE
void tma_gather4(const void* desc_ptr, transac_bar_t &mbar_ptr, void* smem_ptr, int col_idx, int4 row_idxs, int64_t cache_hint) {
uint32_t smem_addr = cute::cast_smem_ptr_to_uint(smem_ptr);
uint32_t mbar_addr = cute::cast_smem_ptr_to_uint(&mbar_ptr);
asm volatile(
"cp.async.bulk.tensor.2d.shared::cta.global.tile::gather4.mbarrier::complete_tx::bytes.cta_group::1.L2::cache_hint [%0], [%1, {%2, %3, %4, %5, %6}], [%7], %8;\n"
:
: "r"(smem_addr), "l"(desc_ptr), "r"(col_idx),
"r"(row_idxs.x), "r"(row_idxs.y), "r"(row_idxs.z), "r"(row_idxs.w),
"r"(mbar_addr), "l"(cache_hint)
: "memory"
);
}
// tma gather4 prefetch (https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-cp-async-bulk-prefetch-tensor)
// Please pay attention that the coordinates of TMA gather4 are int32, which may lead to overflow under some scenarios
CUTE_DEVICE
void tma_gather4_prefetch(const void* desc_ptr, int col_idx, int4 row_idxs, int64_t cache_hint) {
asm volatile(
"cp.async.bulk.prefetch.tensor.2d.L2.global.tile::gather4.L2::cache_hint [%0, {%1, %2, %3, %4, %5}], %6;\n"
:
: "l"(desc_ptr), "r"(col_idx),
"r"(row_idxs.x), "r"(row_idxs.y), "r"(row_idxs.z), "r"(row_idxs.w),
"l"(cache_hint)
);
}
// tma gather4 with cta_group::2, allowing for synchronization across CTAs within a pair of CTAs (https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-cp-async-bulk-tensor)
template<bool USE_CTA0_MBAR = false>
CUTE_DEVICE void tma_gather4_cta_group_2(const void* desc_ptr, transac_bar_t &mbar_ptr, void* smem_ptr, int col_idx, int4 row_idxs, int64_t cache_hint) {
uint32_t smem_addr = cute::cast_smem_ptr_to_uint(smem_ptr);
uint32_t mbar_addr = cute::cast_smem_ptr_to_uint(&mbar_ptr);
if constexpr (USE_CTA0_MBAR) {
mbar_addr &= cute::Sm100MmaPeerBitMask;
}
asm volatile(
"cp.async.bulk.tensor.2d.shared::cta.global.tile::gather4.mbarrier::complete_tx::bytes.cta_group::2.L2::cache_hint [%0], [%1, {%2, %3, %4, %5, %6}], [%7], %8;\n"
:
: "r"(smem_addr), "l"(desc_ptr), "r"(col_idx),
"r"(row_idxs.x), "r"(row_idxs.y), "r"(row_idxs.z), "r"(row_idxs.w),
"r"(mbar_addr), "l"(cache_hint)
: "memory"
);
}
// Vectorized addition for float32 (https://docs.nvidia.com/cuda/parallel-thread-execution/#floating-point-instructions-add)
CUTE_DEVICE
float2 float2_add(const float2 &a, const float2 &b) {
float2 c;
asm volatile(
"add.f32x2 %0, %1, %2;\n"
: "=l"(reinterpret_cast<uint64_t&>(c))
: "l"(reinterpret_cast<uint64_t const&>(a)),
"l"(reinterpret_cast<uint64_t const&>(b))
);
return c;
}
// Vectorized multiplication for float32 (https://docs.nvidia.com/cuda/parallel-thread-execution/#floating-point-instructions-mul)
CUTE_DEVICE
float2 float2_mul(const float2 &a, const float2 &b) {
float2 c;
asm volatile(
"mul.f32x2 %0, %1, %2;\n"
: "=l"(reinterpret_cast<uint64_t&>(c))
: "l"(reinterpret_cast<uint64_t const&>(a)),
"l"(reinterpret_cast<uint64_t const&>(b)));
return c;
}
// Vectorized fused addition-multiplication for float32 (https://docs.nvidia.com/cuda/parallel-thread-execution/#floating-point-instructions-fma)
CUTE_DEVICE
float2 float2_fma(const float2 &a, const float2 &b, const float2 &c) {
// return a*b+c
float2 d;
asm volatile(
"fma.rn.f32x2 %0, %1, %2, %3;\n"
: "=l"(reinterpret_cast<uint64_t&>(d))
: "l"(reinterpret_cast<uint64_t const&>(a)),
"l"(reinterpret_cast<uint64_t const&>(b)),
"l"(reinterpret_cast<uint64_t const&>(c)));
return d;
}
// Vectorized negation for foat32
CUTE_DEVICE
float2 float2_neg(const float2 &a) {
float2 t = {-1.0f, -1.0f};
return float2_mul(a, t);
}
// st.bulk (https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-st-bulk)
CUTE_DEVICE
void st_bulk(void* dst_ptr, int64_t size) {
uint32_t dst_addr = cute::cast_smem_ptr_to_uint(dst_ptr);
asm volatile (
"st.bulk.weak.shared::cta [%0], %1, 0;\n"
:
: "r"(dst_addr), "l"(size)
: "memory"
);
}
struct CUTE_ALIGNAS(16) CLCResponseObj {
// An opaque 16B value
char opaque[16];
};
struct CLCResult {
int is_valid;
int x, y, z;
};
// Issue a CLC try_cancel query (https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-try-cancel)
CUTE_DEVICE
void issue_clc_query(transac_bar_t &bar, CLCResponseObj &response_obj) {
uint32_t response_addr = cute::cast_smem_ptr_to_uint(response_obj.opaque);
uint32_t mbarrier_addr = cute::cast_smem_ptr_to_uint(&bar);
asm volatile(
"clusterlaunchcontrol.try_cancel.async.shared::cta.mbarrier::complete_tx::bytes.b128 [%0], [%1];\n"
:
: "r"(response_addr), "r"(mbarrier_addr)
);
}
// Issue a CLC try_cancel query with .multicast::cluster::all (https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-try-cancel)
CUTE_DEVICE
void issue_clc_query_multicast_cluster_all(transac_bar_t &bar, CLCResponseObj &response_obj) {
uint32_t response_addr = cute::cast_smem_ptr_to_uint(response_obj.opaque);
uint32_t mbarrier_addr = cute::cast_smem_ptr_to_uint(&bar);
asm volatile(
"clusterlaunchcontrol.try_cancel.async.shared::cta.mbarrier::complete_tx::bytes.multicast::cluster::all.b128 [%0], [%1];\n"
:
: "r"(response_addr), "r"(mbarrier_addr)
);
}
// Get the result of a CLC query (https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-query-cancel)
// In this function, we separate get_first_ctaid::x/y/z and hope PTXAS's dead code elimination can remove unnecessary instructions
template<bool USE_LD_ACQUIRE>
CUTE_DEVICE
CLCResult get_clc_query_response(CLCResponseObj &response_obj) {
uint32_t response_addr = cute::cast_smem_ptr_to_uint(&response_obj);
CLCResult result;
#define EMIT_ASM(LD_MODIFIER) \
asm volatile( \
"{\n" \
".reg .pred p1;\n\t" \
".reg .b128 clc_result;\n\t" \
"ld" LD_MODIFIER ".shared.b128 clc_result, [%4];\n\t" \
"clusterlaunchcontrol.query_cancel.is_canceled.pred.b128 p1, clc_result;\n\t" \
"selp.u32 %3, 1, 0, p1;\n\t" \
"@p1 clusterlaunchcontrol.query_cancel.get_first_ctaid::x.b32.b128 %0, clc_result;\n\t" \
"@p1 clusterlaunchcontrol.query_cancel.get_first_ctaid::y.b32.b128 %1, clc_result;\n\t" \
"@p1 clusterlaunchcontrol.query_cancel.get_first_ctaid::z.b32.b128 %2, clc_result;\n\t" \
"}\n" \
: "=r"(result.x), "=r"(result.y), "=r"(result.z), "=r"(result.is_valid) \
: "r"(response_addr) \
: "memory" \
);
if constexpr (USE_LD_ACQUIRE) {
EMIT_ASM(".acquire.cta");
} else {
EMIT_ASM("");
}
return result;
}
// LDG.256 or LDG.256 with non-coherent cache (https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-ld)
// We use macro instead of function here, since we need a multi-level recursive dispatch based on template parameters if using function
// NC_STR should be either "" or ".nc"
// L1_CACHE_HINT_STR should be either "evict_first", "evict_normal", "evict_last", "evict_unchanged", or "no_allocate"
// L2_CACHE_HINT_STR should be either "evict_first", "evict_normal", or "evict_last"
// L2_PREFETCH_SIZE_STR should be either "64B", "128B", or "256B"
#define KU_LDG_256(global_addr, result, NC_STR, L1_CACHE_HINT_STR, L2_CACHE_HINT_STR, L2_PREFETCH_SIZE_STR) \
{ \
static_assert(std::is_pointer_v<decltype(global_addr)> || std::is_array_v<decltype(global_addr)>, "`global_addr` must be a pointer"); \
static_assert(std::is_pointer_v<decltype(result)> || std::is_array_v<decltype(result)>, "`result` must be a pointer"); \
uint64_t* result_as_uint64_ptr = (uint64_t*)(result); \
asm volatile( \
"ld.global" NC_STR ".L1::" L1_CACHE_HINT_STR ".L2::" L2_CACHE_HINT_STR ".L2::" L2_PREFETCH_SIZE_STR ".v4.u64 {%0, %1, %2, %3}, [%4];\n" \
: "=l"(result_as_uint64_ptr[0]), "=l"(result_as_uint64_ptr[1]), \
"=l"(result_as_uint64_ptr[2]), "=l"(result_as_uint64_ptr[3]) \
: "l"(global_addr) \
); \
}
// STG.256 (https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-st)
// L1_CACHE_HINT_STR should be either "evict_first", "evict_normal", "evict_last", "evict_unchanged", or "no_allocate"
// L2_CACHE_HINT_STR should be either "evict_first", "evict_normal", or "evict_last"
#define KU_STG_256(global_addr, src, L1_CACHE_HINT_STR, L2_CACHE_HINT_STR) \
{ \
static_assert(std::is_pointer_v<decltype(global_addr)> || std::is_array_v<decltype(global_addr)>, "`global_addr` must be a pointer"); \
static_assert(std::is_pointer_v<decltype(src)> || std::is_array_v<decltype(src)>, "`src` must be a pointer"); \
uint64_t const* src_as_uint64_ptr = (uint64_t const*)(src); \
asm volatile( \
"st.global.L1::" L1_CACHE_HINT_STR ".L2::" L2_CACHE_HINT_STR ".v4.u64 [%0], {%1, %2, %3, %4};\n" \
: \
: "l"(global_addr), "l"(src_as_uint64_ptr[0]), "l"(src_as_uint64_ptr[1]), \
"l"(src_as_uint64_ptr[2]), "l"(src_as_uint64_ptr[3]) \
); \
}
}
namespace kerutils {
// tcgen05.commit.cta_group::1 (https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen-async-sync-operations-commit)
CUTE_DEVICE
void umma_arrive_noelect(transac_bar_t &bar) {
uint32_t bar_intptr = cute::cast_smem_ptr_to_uint(&bar);
asm volatile(
"tcgen05.commit.cta_group::1.mbarrier::arrive::one.shared::cluster.b64 [%0];\n"
:
:"r"(bar_intptr)
);
}
// tcgen05.commit.cta_group::1, with multicast (https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen-async-sync-operations-commit)
CUTE_DEVICE
void umma_arrive_multicast_noelect(transac_bar_t &bar, uint16_t cta_mask) {
uint32_t bar_intptr = cute::cast_smem_ptr_to_uint(&bar);
asm volatile(
"tcgen05.commit.cta_group::1.mbarrier::arrive::one.shared::cluster.multicast::cluster.b64 [%0], %1;\n"
:
:"r"(bar_intptr), "h"(cta_mask)
);
}
// tcgen05.commit.cta_group::2 (https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen-async-sync-operations-commit)
CUTE_DEVICE
void umma_arrive_2x1SM_noelect(transac_bar_t &bar) {
uint32_t bar_intptr = cute::cast_smem_ptr_to_uint(&bar);
asm volatile(
"tcgen05.commit.cta_group::2.mbarrier::arrive::one.shared::cluster.b64 [%0];\n"
:
:"r"(bar_intptr)
);
}
// tcgen05.commit.cta_group::2, with multicast (https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen-async-sync-operations-commit)
CUTE_DEVICE
void umma_arrive_multicast_2x1SM_noelect(transac_bar_t &bar, uint16_t cta_mask) {
uint32_t bar_intptr = cute::cast_smem_ptr_to_uint(&bar);
asm volatile(
"tcgen05.commit.cta_group::2.mbarrier::arrive::one.shared::cluster.multicast::cluster.b64 [%0], %1;\n"
:
:"r"(bar_intptr), "h"(cta_mask)
);
}
// tcgen05.fence::before_thread_sync (https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-special-sync-operations-fence)
__device__ __forceinline__ void tcgen05_before_thread_sync() {
asm volatile("tcgen05.fence::before_thread_sync;");
}
// tcgen05.fence::after_thread_sync (https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-special-sync-operations-fence)
__device__ __forceinline__ void tcgen05_after_thread_sync() {
asm volatile("tcgen05.fence::after_thread_sync;");
}
// Load from tensor memory, 32 data path lanes, 32-bit pattern, repeated N times. (https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-instructions-tcgen05-ld)
template <int kNumElements>
__device__ __forceinline__
void tmem_ld_32dp32bNx(uint32_t tmem_start, void* data_) {
uint32_t* data = (uint32_t*)data_;
static_assert(kNumElements == 1 || kNumElements == 2 || kNumElements == 4 || kNumElements == 8 || kNumElements == 16 || kNumElements == 32 || kNumElements == 64 || kNumElements == 128, "Invalid kNumElements");
// NOTE The following code crashes VSCode intellisense engine, so we disable it
#ifndef __VSCODE_IDE__
[&]<size_t... Is>(cute::index_sequence<Is...>) {
if constexpr (kNumElements == 1) {
cute::SM100_TMEM_LOAD_32dp32b1x::copy(tmem_start, data[Is]...);
} else if constexpr (kNumElements == 2) {
cute::SM100_TMEM_LOAD_32dp32b2x::copy(tmem_start, data[Is]...);
} else if constexpr (kNumElements == 4) {
cute::SM100_TMEM_LOAD_32dp32b4x::copy(tmem_start, data[Is]...);
} else if constexpr (kNumElements == 8) {
cute::SM100_TMEM_LOAD_32dp32b8x::copy(tmem_start, data[Is]...);
} else if constexpr (kNumElements == 16) {
cute::SM100_TMEM_LOAD_32dp32b16x::copy(tmem_start, data[Is]...);
} else if constexpr (kNumElements == 32) {
cute::SM100_TMEM_LOAD_32dp32b32x::copy(tmem_start, data[Is]...);
} else if constexpr (kNumElements == 64) {
cute::SM100_TMEM_LOAD_32dp32b64x::copy(tmem_start, data[Is]...);
} else if constexpr (kNumElements == 128) {
cute::SM100_TMEM_LOAD_32dp32b128x::copy(tmem_start, data[Is]...);
}
}(cute::make_index_sequence<kNumElements>{});
#endif
}
// Load from tensor memory with column-wise reduction, 32 data path lanes, 32-bit pattern, repeated N times.
// (https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-instructions-tcgen05-ld)
// USE_MAX: true = .max reduction, false = .min reduction
// USE_ABS: true = .abs qualifier (use absolute values for reduction)
// USE_NAN: true = .NaN qualifier (propagate NaN in reduction)
template <int kNumElements, bool USE_MAX, bool USE_ABS, bool USE_NAN>
__device__ __forceinline__
void tmem_ld_red_32dp32bNx(uint32_t tmem_start, void* data_, float& redval) {
uint32_t* data = (uint32_t*)data_;
static_assert(kNumElements == 2 || kNumElements == 4 || kNumElements == 8 ||
kNumElements == 16 || kNumElements == 32 || kNumElements == 64 ||
kNumElements == 128,
"Invalid kNumElements for tcgen05.ld.red (must be power of 2, at least 2)");
#ifndef __VSCODE_IDE__
static constexpr char s_max[] = ".max";
static constexpr char s_min[] = ".min";
static constexpr char s_abs[] = ".abs";
static constexpr char s_nan[] = ".NaN";
static constexpr char s_empty[] = "";
// Operand numbering for each branch:
// outputs: %0..%{N-1} = data[0..N-1] ("=r"), %N = redval ("=f")
// inputs: %{N+1} = tmem_start ("r"), %{N+2} = redop ("C"), %{N+3} = abs ("C"), %{N+4} = nan ("C")
if constexpr (kNumElements == 2) {
// outputs: %0..%1 = data, %2 = redval; inputs: %3 = taddr, %4 = redop, %5 = abs, %6 = nan
asm volatile(
"tcgen05.ld.red.sync.aligned.32x32b.x2%4%5%6.f32"
" {%0, %1}, %2, [%3];\n"
: "=r"(data[0]), "=r"(data[1]),
"=f"(redval)
: "r"(tmem_start),
"C"(USE_MAX ? s_max : s_min),
"C"(USE_ABS ? s_abs : s_empty),
"C"(USE_NAN ? s_nan : s_empty)
);
} else if constexpr (kNumElements == 4) {
// outputs: %0..%3 = data, %4 = redval; inputs: %5 = taddr, %6..%8 = C
asm volatile(
"tcgen05.ld.red.sync.aligned.32x32b.x4%6%7%8.f32"
" {%0, %1, %2, %3}, %4, [%5];\n"
: "=r"(data[0]), "=r"(data[1]), "=r"(data[2]), "=r"(data[3]),
"=f"(redval)
: "r"(tmem_start),
"C"(USE_MAX ? s_max : s_min),
"C"(USE_ABS ? s_abs : s_empty),
"C"(USE_NAN ? s_nan : s_empty)
);
} else if constexpr (kNumElements == 8) {
// outputs: %0..%7 = data, %8 = redval; inputs: %9 = taddr, %10..%12 = C
asm volatile(
"tcgen05.ld.red.sync.aligned.32x32b.x8%10%11%12.f32"
" {%0, %1, %2, %3,"
" %4, %5, %6, %7}, %8, [%9];\n"
: "=r"(data[0]), "=r"(data[1]), "=r"(data[2]), "=r"(data[3]),
"=r"(data[4]), "=r"(data[5]), "=r"(data[6]), "=r"(data[7]),
"=f"(redval)
: "r"(tmem_start),
"C"(USE_MAX ? s_max : s_min),
"C"(USE_ABS ? s_abs : s_empty),
"C"(USE_NAN ? s_nan : s_empty)
);
} else if constexpr (kNumElements == 16) {
// outputs: %0..%15 = data, %16 = redval; inputs: %17 = taddr, %18..%20 = C
asm volatile(
"tcgen05.ld.red.sync.aligned.32x32b.x16%18%19%20.f32"
" {%0, %1, %2, %3,"
" %4, %5, %6, %7,"
" %8, %9, %10, %11,"
" %12, %13, %14, %15}, %16, [%17];\n"
: "=r"(data[0]), "=r"(data[1]), "=r"(data[2]), "=r"(data[3]),
"=r"(data[4]), "=r"(data[5]), "=r"(data[6]), "=r"(data[7]),
"=r"(data[8]), "=r"(data[9]), "=r"(data[10]), "=r"(data[11]),
"=r"(data[12]), "=r"(data[13]), "=r"(data[14]), "=r"(data[15]),
"=f"(redval)
: "r"(tmem_start),
"C"(USE_MAX ? s_max : s_min),
"C"(USE_ABS ? s_abs : s_empty),
"C"(USE_NAN ? s_nan : s_empty)
);
} else if constexpr (kNumElements == 32) {
// outputs: %0..%31 = data, %32 = redval; inputs: %33 = taddr, %34..%36 = C
asm volatile(
"tcgen05.ld.red.sync.aligned.32x32b.x32%34%35%36.f32"
" {%0, %1, %2, %3,"
" %4, %5, %6, %7,"
" %8, %9, %10, %11,"
" %12, %13, %14, %15,"
" %16, %17, %18, %19,"
" %20, %21, %22, %23,"
" %24, %25, %26, %27,"
" %28, %29, %30, %31}, %32, [%33];\n"
: "=r"(data[0]), "=r"(data[1]), "=r"(data[2]), "=r"(data[3]),
"=r"(data[4]), "=r"(data[5]), "=r"(data[6]), "=r"(data[7]),
"=r"(data[8]), "=r"(data[9]), "=r"(data[10]), "=r"(data[11]),
"=r"(data[12]), "=r"(data[13]), "=r"(data[14]), "=r"(data[15]),
"=r"(data[16]), "=r"(data[17]), "=r"(data[18]), "=r"(data[19]),
"=r"(data[20]), "=r"(data[21]), "=r"(data[22]), "=r"(data[23]),
"=r"(data[24]), "=r"(data[25]), "=r"(data[26]), "=r"(data[27]),
"=r"(data[28]), "=r"(data[29]), "=r"(data[30]), "=r"(data[31]),
"=f"(redval)
: "r"(tmem_start),
"C"(USE_MAX ? s_max : s_min),
"C"(USE_ABS ? s_abs : s_empty),
"C"(USE_NAN ? s_nan : s_empty)
);
} else if constexpr (kNumElements == 64) {
// outputs: %0..%63 = data, %64 = redval; inputs: %65 = taddr, %66..%68 = C
asm volatile(
"tcgen05.ld.red.sync.aligned.32x32b.x64%66%67%68.f32"
" {%0, %1, %2, %3,"
" %4, %5, %6, %7,"
" %8, %9, %10, %11,"
" %12, %13, %14, %15,"
" %16, %17, %18, %19,"
" %20, %21, %22, %23,"
" %24, %25, %26, %27,"
" %28, %29, %30, %31,"
" %32, %33, %34, %35,"
" %36, %37, %38, %39,"
" %40, %41, %42, %43,"
" %44, %45, %46, %47,"
" %48, %49, %50, %51,"
" %52, %53, %54, %55,"
" %56, %57, %58, %59,"
" %60, %61, %62, %63}, %64, [%65];\n"
: "=r"(data[0]), "=r"(data[1]), "=r"(data[2]), "=r"(data[3]),
"=r"(data[4]), "=r"(data[5]), "=r"(data[6]), "=r"(data[7]),
"=r"(data[8]), "=r"(data[9]), "=r"(data[10]), "=r"(data[11]),
"=r"(data[12]), "=r"(data[13]), "=r"(data[14]), "=r"(data[15]),
"=r"(data[16]), "=r"(data[17]), "=r"(data[18]), "=r"(data[19]),
"=r"(data[20]), "=r"(data[21]), "=r"(data[22]), "=r"(data[23]),
"=r"(data[24]), "=r"(data[25]), "=r"(data[26]), "=r"(data[27]),
"=r"(data[28]), "=r"(data[29]), "=r"(data[30]), "=r"(data[31]),
"=r"(data[32]), "=r"(data[33]), "=r"(data[34]), "=r"(data[35]),
"=r"(data[36]), "=r"(data[37]), "=r"(data[38]), "=r"(data[39]),
"=r"(data[40]), "=r"(data[41]), "=r"(data[42]), "=r"(data[43]),
"=r"(data[44]), "=r"(data[45]), "=r"(data[46]), "=r"(data[47]),
"=r"(data[48]), "=r"(data[49]), "=r"(data[50]), "=r"(data[51]),
"=r"(data[52]), "=r"(data[53]), "=r"(data[54]), "=r"(data[55]),
"=r"(data[56]), "=r"(data[57]), "=r"(data[58]), "=r"(data[59]),
"=r"(data[60]), "=r"(data[61]), "=r"(data[62]), "=r"(data[63]),
"=f"(redval)
: "r"(tmem_start),
"C"(USE_MAX ? s_max : s_min),
"C"(USE_ABS ? s_abs : s_empty),
"C"(USE_NAN ? s_nan : s_empty)
);
} else if constexpr (kNumElements == 128) {
// outputs: %0..%127 = data, %128 = redval; inputs: %129 = taddr, %130..%132 = C
asm volatile(
"tcgen05.ld.red.sync.aligned.32x32b.x128%130%131%132.f32"
" {%0, %1, %2, %3,"
" %4, %5, %6, %7,"
" %8, %9, %10, %11,"
" %12, %13, %14, %15,"
" %16, %17, %18, %19,"
" %20, %21, %22, %23,"
" %24, %25, %26, %27,"
" %28, %29, %30, %31,"
" %32, %33, %34, %35,"
" %36, %37, %38, %39,"
" %40, %41, %42, %43,"
" %44, %45, %46, %47,"
" %48, %49, %50, %51,"
" %52, %53, %54, %55,"
" %56, %57, %58, %59,"
" %60, %61, %62, %63,"
" %64, %65, %66, %67,"
" %68, %69, %70, %71,"
" %72, %73, %74, %75,"
" %76, %77, %78, %79,"
" %80, %81, %82, %83,"
" %84, %85, %86, %87,"
" %88, %89, %90, %91,"
" %92, %93, %94, %95,"
" %96, %97, %98, %99,"
" %100, %101, %102, %103,"
" %104, %105, %106, %107,"
" %108, %109, %110, %111,"
" %112, %113, %114, %115,"
" %116, %117, %118, %119,"
" %120, %121, %122, %123,"
" %124, %125, %126, %127}, %128, [%129];\n"
: "=r"(data[0]), "=r"(data[1]), "=r"(data[2]), "=r"(data[3]),
"=r"(data[4]), "=r"(data[5]), "=r"(data[6]), "=r"(data[7]),
"=r"(data[8]), "=r"(data[9]), "=r"(data[10]), "=r"(data[11]),
"=r"(data[12]), "=r"(data[13]), "=r"(data[14]), "=r"(data[15]),
"=r"(data[16]), "=r"(data[17]), "=r"(data[18]), "=r"(data[19]),
"=r"(data[20]), "=r"(data[21]), "=r"(data[22]), "=r"(data[23]),
"=r"(data[24]), "=r"(data[25]), "=r"(data[26]), "=r"(data[27]),
"=r"(data[28]), "=r"(data[29]), "=r"(data[30]), "=r"(data[31]),
"=r"(data[32]), "=r"(data[33]), "=r"(data[34]), "=r"(data[35]),
"=r"(data[36]), "=r"(data[37]), "=r"(data[38]), "=r"(data[39]),
"=r"(data[40]), "=r"(data[41]), "=r"(data[42]), "=r"(data[43]),
"=r"(data[44]), "=r"(data[45]), "=r"(data[46]), "=r"(data[47]),
"=r"(data[48]), "=r"(data[49]), "=r"(data[50]), "=r"(data[51]),
"=r"(data[52]), "=r"(data[53]), "=r"(data[54]), "=r"(data[55]),
"=r"(data[56]), "=r"(data[57]), "=r"(data[58]), "=r"(data[59]),
"=r"(data[60]), "=r"(data[61]), "=r"(data[62]), "=r"(data[63]),
"=r"(data[64]), "=r"(data[65]), "=r"(data[66]), "=r"(data[67]),
"=r"(data[68]), "=r"(data[69]), "=r"(data[70]), "=r"(data[71]),
"=r"(data[72]), "=r"(data[73]), "=r"(data[74]), "=r"(data[75]),
"=r"(data[76]), "=r"(data[77]), "=r"(data[78]), "=r"(data[79]),
"=r"(data[80]), "=r"(data[81]), "=r"(data[82]), "=r"(data[83]),
"=r"(data[84]), "=r"(data[85]), "=r"(data[86]), "=r"(data[87]),
"=r"(data[88]), "=r"(data[89]), "=r"(data[90]), "=r"(data[91]),
"=r"(data[92]), "=r"(data[93]), "=r"(data[94]), "=r"(data[95]),
"=r"(data[96]), "=r"(data[97]), "=r"(data[98]), "=r"(data[99]),
"=r"(data[100]), "=r"(data[101]), "=r"(data[102]), "=r"(data[103]),
"=r"(data[104]), "=r"(data[105]), "=r"(data[106]), "=r"(data[107]),
"=r"(data[108]), "=r"(data[109]), "=r"(data[110]), "=r"(data[111]),
"=r"(data[112]), "=r"(data[113]), "=r"(data[114]), "=r"(data[115]),
"=r"(data[116]), "=r"(data[117]), "=r"(data[118]), "=r"(data[119]),
"=r"(data[120]), "=r"(data[121]), "=r"(data[122]), "=r"(data[123]),
"=r"(data[124]), "=r"(data[125]), "=r"(data[126]), "=r"(data[127]),
"=f"(redval)
: "r"(tmem_start),
"C"(USE_MAX ? s_max : s_min),
"C"(USE_ABS ? s_abs : s_empty),
"C"(USE_NAN ? s_nan : s_empty)
);
}
#endif
}
// Load from tensor memory, 16 data path lanes, 128-bit pattern, repeated N times. (https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-instructions-tcgen05-ld)
template <int kNumReplications>
__device__ __forceinline__
void tmem_ld_16dp128bNx(uint32_t tmem_start, void* data_) {
uint32_t* data = (uint32_t*)data_;
static_assert(kNumReplications == 1 || kNumReplications == 2 || kNumReplications == 4 || kNumReplications == 8 || kNumReplications == 16 || kNumReplications == 32 || kNumReplications == 64, "Invalid kNumReplications");
#ifndef __VSCODE_IDE__
[&]<size_t... Is>(cute::index_sequence<Is...>) {
if constexpr (kNumReplications == 1) {
cute::SM100_TMEM_LOAD_16dp128b1x::copy(tmem_start, data[Is]...);
} else if constexpr (kNumReplications == 2) {
cute::SM100_TMEM_LOAD_16dp128b2x::copy(tmem_start, data[Is]...);
} else if constexpr (kNumReplications == 4) {
cute::SM100_TMEM_LOAD_16dp128b4x::copy(tmem_start, data[Is]...);
} else if constexpr (kNumReplications == 8) {
cute::SM100_TMEM_LOAD_16dp128b8x::copy(tmem_start, data[Is]...);
} else if constexpr (kNumReplications == 16) {
cute::SM100_TMEM_LOAD_16dp128b16x::copy(tmem_start, data[Is]...);
} else if constexpr (kNumReplications == 32) {
cute::SM100_TMEM_LOAD_16dp128b32x::copy(tmem_start, data[Is]...);
} else if constexpr (kNumReplications == 64) {
cute::SM100_TMEM_LOAD_16dp128b64x::copy(tmem_start, data[Is]...);
}
}(cute::make_index_sequence<kNumReplications*2>{});
#endif
}
// Load from tensor memory, 16 data path lanes, 256-bit pattern, repeated N times. (https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-instructions-tcgen05-ld)
template <int kNumReplications>
__device__ __forceinline__
void tmem_ld_16dp256bNx(uint32_t tmem_start, void* data_) {
uint32_t* data = (uint32_t*)data_;
static_assert(kNumReplications == 1 || kNumReplications == 2 || kNumReplications == 4 || kNumReplications == 8 || kNumReplications == 16 || kNumReplications == 32, "Invalid kNumReplications");
#ifndef __VSCODE_IDE__
[&]<size_t... Is>(cute::index_sequence<Is...>) {
if constexpr (kNumReplications == 1) {
cute::SM100_TMEM_LOAD_16dp256b1x::copy(tmem_start, data[Is]...);
} else if constexpr (kNumReplications == 2) {
cute::SM100_TMEM_LOAD_16dp256b2x::copy(tmem_start, data[Is]...);
} else if constexpr (kNumReplications == 4) {
cute::SM100_TMEM_LOAD_16dp256b4x::copy(tmem_start, data[Is]...);
} else if constexpr (kNumReplications == 8) {
cute::SM100_TMEM_LOAD_16dp256b8x::copy(tmem_start, data[Is]...);
} else if constexpr (kNumReplications == 16) {
cute::SM100_TMEM_LOAD_16dp256b16x::copy(tmem_start, data[Is]...);
} else if constexpr (kNumReplications == 32) {
cute::SM100_TMEM_LOAD_16dp256b32x::copy(tmem_start, data[Is]...);
}
}(cute::make_index_sequence<kNumReplications*4>{});
#endif
}
// Store into tensor memory, 32 data path lanes, 32-bit pattern, repeated N times. (https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-instructions-tcgen05-st)
template <int kNumElements>
__device__ __forceinline__
void tmem_st_32dp32bNx(uint32_t tmem_start, void const* data_) {
uint32_t const* data = (uint32_t const*)data_;
static_assert(kNumElements == 1 || kNumElements == 2 || kNumElements == 4 || kNumElements == 8 || kNumElements == 16 || kNumElements == 32 || kNumElements == 64 || kNumElements == 128, "Invalid kNumElements");
#ifndef __VSCODE_IDE__
[&]<size_t... Is>(cute::index_sequence<Is...>) {
if constexpr (kNumElements == 1) {
cute::SM100_TMEM_STORE_32dp32b1x::copy(data[Is]..., tmem_start);
} else if constexpr (kNumElements == 2) {
cute::SM100_TMEM_STORE_32dp32b2x::copy(data[Is]..., tmem_start);
} else if constexpr (kNumElements == 4) {
cute::SM100_TMEM_STORE_32dp32b4x::copy(data[Is]..., tmem_start);
} else if constexpr (kNumElements == 8) {
cute::SM100_TMEM_STORE_32dp32b8x::copy(data[Is]..., tmem_start);
} else if constexpr (kNumElements == 16) {
cute::SM100_TMEM_STORE_32dp32b16x::copy(data[Is]..., tmem_start);
} else if constexpr (kNumElements == 32) {
cute::SM100_TMEM_STORE_32dp32b32x::copy(data[Is]..., tmem_start);
} else if constexpr (kNumElements == 64) {
cute::SM100_TMEM_STORE_32dp32b64x::copy(data[Is]..., tmem_start);
} else if constexpr (kNumElements == 128) {
cute::SM100_TMEM_STORE_32dp32b128x::copy(data[Is]..., tmem_start);
}
}(cute::make_index_sequence<kNumElements>{});
#endif
}
// Decompose a non-power-of-2 replication count into a sequence of power-of-2 calls.
//
// This helper splits kNumReplications into its constituent powers of 2 (from highest to lowest)
// and issues one call to tmem_ld_st_fn.template operator()<power>() for each, automatically
// advancing the tmem address by kNumElemsPerUnit columns and the data pointer by kNumElemsPerUnit
// uint32_t elements per unit.
//
// Template parameters:
// kNumReplications — total number of replications (may be non-power-of-2)
// kNumElemsPerUnit — number of tmem columns (and uint32_t data elements) consumed per unit
// (e.g. 1 for 32dp32b, 2 for 16dp128b, 4 for 16dp256b)
// F — a callable with a template operator(): f.template operator()<N>(uint32_t, void*)
//
// Usage example (with a template lambda):
// tmem_ld_st_decomposed<6, 1>(tmem_start, data_ptr,
// []<int N>(uint32_t ts, void* d) { tmem_ld_32dp32bNx<N>(ts, d); });
// // Equivalent to:
// // tmem_ld_32dp32bNx<4>(tmem_start, data_ptr);
// // tmem_ld_32dp32bNx<2>(tmem_start + 4, (void*)((uint32_t*)data_ptr + 4));
template<int kNumReplications, int kNumElemsPerUnit, typename F>
__device__ __forceinline__
void tmem_ld_st_decomposed(uint32_t tmem_start, void* data, F&& tmem_ld_st_fn) {
if constexpr (kNumReplications == 0) {
// Base case: nothing to do
return;
} else {
// Largest power-of-2 <= kNumReplications
constexpr int kHighBit = 1 << (31 - __builtin_clz(kNumReplications));
tmem_ld_st_fn.template operator()<kHighBit>(tmem_start, data);
// Recurse on the remainder
constexpr int kRemaining = kNumReplications - kHighBit;
if constexpr (kRemaining > 0) {
tmem_ld_st_decomposed<kRemaining, kNumElemsPerUnit>(
tmem_start + kHighBit * kNumElemsPerUnit,
(void*)((uint32_t*)data + kHighBit * kNumElemsPerUnit),
tmem_ld_st_fn
);
}
}
}
}
@@ -0,0 +1,55 @@
#pragma once
#include "kerutils/device/cuda/common.h"
#include "kerutils/device/cuda/sm80/intrinsics.cuh"
namespace kerutils {
// Retrieve the value of `%smid` and check its range
CUTE_DEVICE
uint32_t get_sm_id_with_range_check(uint32_t num_physical_sms) {
uint32_t sm_id = get_sm_id();
if (!(sm_id < num_physical_sms)) {
trap();
}
return sm_id;
}
#ifndef KU_TRAP_ONLY_DEVICE_ASSERT
#define KU_TRAP_ONLY_DEVICE_ASSERT(cond) \
do { \
if (not (cond)) \
asm("trap;"); \
} while (0)
#endif
// Construct a `float2` from a single `float` by duplicating the value
CUTE_DEVICE
float2 float2float2(const float &x) {
return float2 {x, x};
}
CUTE_DEVICE
void st_shared(void* ptr, __int128_t val) {
asm volatile("st.shared.b128 [%0], %1;" :: "r"(cute::cast_smem_ptr_to_uint(ptr)), "q"(val));
}
CUTE_DEVICE
void st_shared(void* ptr, float4 val) {
st_shared(ptr, *(__int128_t*)&val);
}
CUTE_DEVICE
__int128_t ld_shared(const void* ptr) {
__int128_t val;
asm volatile("ld.shared.b128 %0, [%1];" : "=q"(val) : "r"(cute::cast_smem_ptr_to_uint(ptr)));
return val;
}
CUTE_DEVICE
float4 ld_shared_float4(const void* ptr) {
__int128_t temp = ld_shared(ptr);
return *(float4*)&temp;
}
}
@@ -0,0 +1,160 @@
#pragma once
#include "kerutils/device/cuda/common.h"
namespace kerutils {
// cp.async.cg (cache global) with prefetch and predicate (https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-cp-async)
template<PrefetchSize PREFETCH_SIZE=PrefetchSize::B128>
CUTE_DEVICE
void cp_async_cacheglobal(const void* src, void* dst, bool pred=true) {
uint32_t dst_addr = cute::cast_smem_ptr_to_uint(dst);
if constexpr (PREFETCH_SIZE == PrefetchSize::B64) {
asm volatile("cp.async.cg.shared.global.L2::64B [%0], [%1], 16, %2;\n"
:: "r"(dst_addr),
"l"(src),
"r"(pred?16:0));
} else if constexpr (PREFETCH_SIZE == PrefetchSize::B128) {
asm volatile("cp.async.cg.shared.global.L2::128B [%0], [%1], 16, %2;\n"
:: "r"(dst_addr),
"l"(src),
"r"(pred?16:0));
} else if constexpr (PREFETCH_SIZE == PrefetchSize::B256) {
asm volatile("cp.async.cg.shared.global.L2::256B [%0], [%1], 16, %2;\n"
:: "r"(dst_addr),
"l"(src),
"r"(pred?16:0));
} else {
static_assert(PREFETCH_SIZE == PrefetchSize::B64 ||
PREFETCH_SIZE == PrefetchSize::B128 ||
PREFETCH_SIZE == PrefetchSize::B256,
"Unsupported prefetch size for cp_async_cacheglobal.");
}
}
// Create fraction-based cache policy (https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-createpolicy)
template<CacheHint PRIMARY_PRIORITY, CacheHint SECONDARY_PRIORITY>
CUTE_DEVICE
int64_t create_fraction_based_cache_policy(float fraction = 1.0f) {
int64_t result;
#define EMIT(PRIMARY_PRIORITY_STR, SECONDARY_PRIORITY_STR) \
asm volatile( \
"createpolicy.fractional.L2::" PRIMARY_PRIORITY_STR ".L2::" SECONDARY_PRIORITY_STR ".b64 %0, %1;\n" \
: "=l"(result) \
: "f"(fraction) \
);
#define EMIT2(PRIMARY_PRIORITY_STR) \
{ \
if constexpr (SECONDARY_PRIORITY == CacheHint::EVICT_FIRST) { \
EMIT(PRIMARY_PRIORITY_STR, "evict_first") \
} else if constexpr (SECONDARY_PRIORITY == CacheHint::EVICT_UNCHANGED) { \
EMIT(PRIMARY_PRIORITY_STR, "evict_unchanged") \
} else { \
static_assert(SECONDARY_PRIORITY == CacheHint::EVICT_FIRST || \
SECONDARY_PRIORITY == CacheHint::EVICT_UNCHANGED, \
"Unsupported secondary cache hint for create_fraction_based_cache_policy."); \
} \
}
if constexpr (PRIMARY_PRIORITY == CacheHint::EVICT_FIRST) {
EMIT2("evict_first");
} else if constexpr (PRIMARY_PRIORITY == CacheHint::EVICT_NORMAL) {
EMIT2("evict_normal");
} else if constexpr (PRIMARY_PRIORITY == CacheHint::EVICT_LAST) {
EMIT2("evict_last");
} else if constexpr (PRIMARY_PRIORITY == CacheHint::EVICT_UNCHANGED) {
EMIT2("evict_unchanged");
} else {
static_assert(PRIMARY_PRIORITY == CacheHint::EVICT_FIRST ||
PRIMARY_PRIORITY == CacheHint::EVICT_NORMAL ||
PRIMARY_PRIORITY == CacheHint::EVICT_LAST ||
PRIMARY_PRIORITY == CacheHint::EVICT_UNCHANGED,
"Unsupported primary cache hint for create_fraction_based_cache_policy.");
}
#undef EMIT
#undef EMIT2
return result;
}
// Create a simple cache policy (equivalent to create_fraction_based_cache_policy(1.0f))
// The same as cute::TMA::CacheHintSmXX
template<CacheHint CACHE_HINT>
CUTE_DEVICE
constexpr int64_t create_simple_cache_policy() {
if constexpr (CACHE_HINT == CacheHint::EVICT_FIRST) {
return 0x12F0000000000000; // Result of createpolicy.fractional.L2::evict_first.b64
} else if constexpr (CACHE_HINT == CacheHint::EVICT_NORMAL) {
return 0x1000000000000000; // Copied from CuTe. Unsure about the exact meaning. (TODO Change to 0x16F0000000000000?)
} else if constexpr (CACHE_HINT == CacheHint::EVICT_LAST) {
return 0x14F0000000000000; // Result of createpolicy.fractional.L2::evict_last.b64
} else {
static_assert(CACHE_HINT == CacheHint::EVICT_FIRST ||
CACHE_HINT == CacheHint::EVICT_NORMAL ||
CACHE_HINT == CacheHint::EVICT_LAST,
"Unsupported cache hint for create_simple_cache_policy.");
}
}
// AtomicAdd (https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-red)
CUTE_DEVICE
void atomicadd_f32_with_policy_and_pred(void* global_addr, const float &data, int64_t cache_policy, uint32_t pred = true) {
asm volatile(
"{\n\t"
".reg .pred p;\n\t"
"setp.eq.u32 p, %3, 1;\n\t"
"@p red.relaxed.gpu.global.add.L2::cache_hint.f32 [%1], %0, %2; \n\t"
"}"
:
: "f"(data),
"l"((int64_t)global_addr), "l"(cache_policy), "r"(pred)
);
}
// Get the id of the current SM
// About %smid (https://docs.nvidia.com/cuda/parallel-thread-execution/#special-registers-smid): PTX document says that %smid ranges from 0 to %nsmid-1, while "The SM identifier numbering is not guaranteed to be contiguous, so %nsmid may be larger than the physical number of SMs in the device.". However, result shows that, at least for sm90 and sm100, %nsmid is the number of physical SMs - 1. For the sake of safety, I recommend you to check the return of get_sm_id manually or call `get_sm_id_with_range_check()` defined in `device/sm80/helpers.cuh`.
// Besides, PTX document also says that this number may change due to preemption, but currently this never happens according to [DATEN GELÖSCHT]
CUTE_DEVICE
uint32_t get_sm_id() {
uint32_t ret;
asm volatile("mov.u32 %0, %%smid;\n" : "=r"(ret));
return ret;
}
// trap (https://docs.nvidia.com/cuda/parallel-thread-execution/#miscellaneous-instructions-trap)
CUTE_DEVICE
void trap() {
asm volatile("trap;\n");
}
// LDG.128 or LDG.128 with non-coherent cache (https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-ld)
// We use macro instead of function here, since we need a multi-level recursive dispatch based on template parameters if using function
// NC_STR should be either "" or ".nc"
// L1_CACHE_HINT_STR should be either "evict_first", "evict_normal", "evict_last", "evict_unchanged", or "no_allocate"
// L2_PREFETCH_SIZE_STR should be either "64B", "128B", or "256B"
// L2 cache hint is not supported since it's only supported for LDG.256
#define KU_LDG_128(global_addr, result, NC_STR, L1_CACHE_HINT_STR, L2_PREFETCH_SIZE_STR) \
{ \
static_assert(std::is_pointer_v<decltype(global_addr)> || std::is_array_v<decltype(global_addr)>, "`global_addr` must be a pointer"); \
static_assert(std::is_pointer_v<decltype(result)> || std::is_array_v<decltype(result)>, "`result` must be a pointer"); \
uint64_t* result_as_uint64_ptr = (uint64_t*)(result); \
asm volatile( \
"ld.global" NC_STR ".L1::" L1_CACHE_HINT_STR ".L2::" L2_PREFETCH_SIZE_STR ".v2.u64 {%0, %1}, [%2];\n" \
: "=l"(result_as_uint64_ptr[0]), "=l"(result_as_uint64_ptr[1]) \
: "l"(global_addr) \
); \
}
// STG.128 (https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-st)
// L2_CACHE_HINT_STR should be ".L2::evict_XXX" (only available on sm100+) or ""
#define KU_STG_128(global_addr, src, L1_CACHE_HINT_STR, L2_CACHE_HINT_STR) \
{ \
static_assert(std::is_pointer_v<decltype(global_addr)> || std::is_array_v<decltype(global_addr)>, "`global_addr` must be a pointer"); \
static_assert(std::is_pointer_v<decltype(src)> || std::is_array_v<decltype(src)>, "`src` must be a pointer"); \
uint64_t const* src_as_uint64_ptr = (uint64_t const*)(src); \
asm volatile( \
"st.global.L1::" L1_CACHE_HINT_STR L2_CACHE_HINT_STR ".v2.u64 [%0], {%1, %2};\n" \
: \
: "l"(global_addr), "l"(src_as_uint64_ptr[0]), "l"(src_as_uint64_ptr[1]) \
); \
}
}
@@ -0,0 +1,107 @@
#pragma once
#include "kerutils/device/cuda/common.h"
namespace kerutils {
// st.async (https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-st-async)
template<typename T>
CUTE_DEVICE
static void st_async(void* dst_ptr, const T& data, transac_bar_t &mbar) {
static_assert(sizeof(T) == 16, "Data type must be 16 bytes (128 bits) for st_async.");
long2 data_long2 = *reinterpret_cast<const long2*>(&data);
uint32_t dst_addr = cute::cast_smem_ptr_to_uint(dst_ptr);
uint32_t mbar_addr = cute::cast_smem_ptr_to_uint(&mbar);
asm volatile (
"st.async.weak.shared::cluster.mbarrier::complete_tx::bytes.v2.s64 [%0], {%1, %2}, [%3]; \n"
:
: "r"(dst_addr), "l"(data_long2.x), "l"(data_long2.y), "r"(mbar_addr)
);
}
static constexpr int PEER_ADDR_MASK = 16777216;
// Given an address in the current CTA, return the corresponding address in the peer CTA
template<typename T>
CUTE_DEVICE
T* get_peer_addr(const T* p) {
return (T*)((int64_t)(p) ^ PEER_ADDR_MASK);
}
// Given an address in the current CTA, return the corresponding address in the peer CTA (if the current CTA_id%2 == 1) or the address itself (if CTA_id%2 == 0)
template<typename T>
CUTE_DEVICE
T* get_cta0_addr(const T* p) {
constexpr int CTA0_ADDR_MASK = 0xFEFFFFFF;
return (T*)((int64_t)(p) & CTA0_ADDR_MASK);
}
// TMA bulk reduce add (cp.reduce.async.bulk), shared to global, float32, add. (https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-cp-reduce-async-bulk)
CUTE_DEVICE
void tma_bulk_reduce_add(void const* src_ptr, void* dst_ptr, int32_t store_bytes) {
uint32_t smem_int_ptr = cute::cast_smem_ptr_to_uint(src_ptr);
asm volatile("cp.reduce.async.bulk.global.shared::cta.bulk_group.add.f32 [%0], [%1], %2;\n"
:
: "l"(dst_ptr), "r"(smem_int_ptr), "r"(store_bytes)
: "memory");
}
// Cluster barrier arrive with .release modifier. (https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-barrier-cluster)
CUTE_DEVICE
void barrier_cluster_arrive_release() {
asm volatile("barrier.cluster.arrive.release;" : : : "memory");
}
// Cluster barrier arrive with .relaxed modifier. (https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-barrier-cluster)
CUTE_DEVICE
void barrier_cluster_arrive_relaxed() {
asm volatile("barrier.cluster.arrive.relaxed;" : : :);
}
// Cluster barrier wait with .acquire modifier. (https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-barrier-cluster)
CUTE_DEVICE
void barrier_cluster_wait_acquire() {
asm volatile("barrier.cluster.wait.acquire;" : : : "memory");
}
// mbarrier.arrive with .relaxed.cluster qualifier (https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-mbarrier-arrive)
CUTE_DEVICE
void mbarrier_arrive_relaxed_cluster(transac_bar_t &mbar) {
uint32_t smem_addr = cute::cast_smem_ptr_to_uint(&mbar);
asm volatile(
"{\n\t"
"mbarrier.arrive.relaxed.cluster.shared::cta.b64 _, [%0];\n\t"
"}"
:
: "r"(smem_addr));
}
// AtomicAdd with v4.f32 type (https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-red)
CUTE_DEVICE
void atomicadd_f32x4_with_policy_and_pred(void* global_addr, const float4 &data, int64_t cache_policy, uint32_t pred = true) {
asm volatile(
"{\n\t"
".reg .pred p;\n\t"
"setp.eq.u32 p, %6, 1;\n\t"
"@p red.relaxed.gpu.global.add.L2::cache_hint.v4.f32 [%4], {%0, %1, %2, %3}, %5; \n\t"
"}"
:
: "f"(data.x), "f"(data.y), "f"(data.z), "f"(data.w),
"l"((int64_t)global_addr), "l"(cache_policy), "r"(pred)
);
}
// cp.async.bulk, from .shared::cta to .shared::cluster (https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-cp-async-bulk)
CUTE_DEVICE
void cp_async_bulk_shared_cta_to_shared_cluster(void* dst_ptr, const void* src_ptr, int32_t load_bytes, transac_bar_t &mbar) {
uint32_t dst_smem_addr = cute::cast_smem_ptr_to_uint(dst_ptr);
uint32_t src_smem_addr = cute::cast_smem_ptr_to_uint(src_ptr);
uint32_t mbar_addr = cute::cast_smem_ptr_to_uint(&mbar);
asm volatile(
"cp.async.bulk.shared::cluster.shared::cta.mbarrier::complete_tx::bytes [%0], [%1], %2, [%3]; \n"
:
: "r"(dst_smem_addr), "r"(src_smem_addr), "r"(load_bytes), "r"(mbar_addr)
);
}
}
@@ -0,0 +1,11 @@
#pragma once
#include "kerutils/common/common.h"
#ifdef KERUTILS_IS_BUILD_ON_CUDA
#include "cuda/common.h"
#include "cuda/sm80/intrinsics.cuh"
#include "cuda/sm80/helpers.cuh"
#include "cuda/sm90/intrinsics.cuh"
#include "cuda/sm100/intrinsics.cuh"
#endif
@@ -0,0 +1,282 @@
#pragma once
#include <cstdio>
#include <exception>
#include <memory>
#include <string>
#include <utility>
#include <sstream>
#include <vector>
#ifdef KERUTILS_IS_BUILD_ON_CUDA
#include <cuda_runtime_api.h>
#include <cudaTypedefs.h>
#endif
#include "kerutils/common/common.h"
namespace kerutils {
class KUException final : public std::exception {
std::string message = {};
public:
template<typename... Args>
explicit KUException(const char *name, const char* file, const int line, Args&&... args) {
std::ostringstream oss;
oss << name << " error (" << file << ":" << line << "): ";
(oss << ... << args);
message = oss.str();
}
const char *what() const noexcept override {
return message.c_str();
}
};
#define THROW_KU_EXCEPTION(name, ...) \
throw kerutils::KUException(name, __FILE__, __LINE__, __VA_ARGS__)
// This `KU_ASSERT` is triggered no matter if the code is compiled with `-DNDEBUG` or not.
#define KU_ASSERT(cond, ...) \
do { \
if (not (cond)) { \
char _ku_buf[1024]; \
int _ku_len = snprintf(_ku_buf, sizeof(_ku_buf), \
"Assertion `%s` failed (%s:%d)", #cond, __FILE__, __LINE__); \
__VA_OPT__( \
_ku_len += snprintf(_ku_buf + _ku_len, sizeof(_ku_buf) - _ku_len, \
": " __VA_ARGS__); \
) \
fprintf(stderr, "%s\n", _ku_buf); \
THROW_KU_EXCEPTION("Assertion", _ku_buf); \
} \
} while(0)
#ifdef KERUTILS_IS_BUILD_ON_CUDA
#define KU_CUDA_CHECK(call) \
do { \
cudaError_t status_ = (call); \
if (status_ != cudaSuccess) { \
char _ku_buf[1024]; \
snprintf(_ku_buf, sizeof(_ku_buf), "CUDA error (%s:%d): %s", __FILE__, __LINE__, cudaGetErrorString(status_)); \
fprintf(stderr, "%s\n", _ku_buf); \
THROW_KU_EXCEPTION("CUDA", _ku_buf); \
} \
} while(0)
#define KU_CUTLASS_CHECK(call) \
do { \
cutlass::Status status_ = (call); \
if (status_ != cutlass::Status::kSuccess) { \
char _ku_buf[1024]; \
snprintf(_ku_buf, sizeof(_ku_buf), "CUTLASS error (%s:%d): %d", __FILE__, __LINE__, static_cast<int>(status_)); \
fprintf(stderr, "%s\n", _ku_buf); \
THROW_KU_EXCEPTION("CUTLASS", _ku_buf); \
} \
} while(0)
#define KU_CHECK_KERNEL_LAUNCH() KU_CUDA_CHECK(cudaGetLastError())
template<typename T>
inline __host__ __device__ constexpr T ceil_div(const T &a, const T &b) {
return (a + b - 1) / b;
}
template<typename T>
inline __host__ __device__ constexpr T ceil(const T &a, const T &b) {
return (a + b - 1) / b * b;
}
template<typename T, T LOWER_BOUND = 1>
inline __host__ __device__ constexpr T find_next_power_of_2(const T& x) {
if (x <= LOWER_BOUND)
return LOWER_BOUND;
return find_next_power_of_2<T, LOWER_BOUND*2>(x);
}
// A wrapper for make_tensor_map
static inline CUtensorMap make_tensor_map(
const std::vector<uint64_t> &size,
const std::vector<uint64_t> &strides, // PAY ATTENTION: In BYTES
const std::vector<uint32_t> &box_size,
void* global_ptr,
CUtensorMapDataType data_type,
CUtensorMapSwizzle swizzle_mode,
CUtensorMapL2promotion l2_promotion,
CUtensorMapInterleave interleave_mode = CUtensorMapInterleave::CU_TENSOR_MAP_INTERLEAVE_NONE,
CUtensorMapFloatOOBfill oob_fill = CUtensorMapFloatOOBfill::CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE,
const std::vector<uint32_t> &element_strides_ = {}
) {
int dim = size.size();
KU_ASSERT(dim >= 1);
std::vector<uint32_t> element_strides;
if (element_strides_.empty()) {
for (int i = 0; i < dim; ++i)
element_strides.push_back(1);
} else {
element_strides = element_strides_;
}
KU_ASSERT(strides.size() == (uint32_t)dim-1 && box_size.size() == (uint32_t)dim && element_strides.size() == (uint32_t)dim);
auto call_cuTensorMapEncodeTiled = [&]<typename... Args>(Args... args) {
cudaDriverEntryPointQueryResult cuda_status;
void* pfn = nullptr;
#if (__CUDACC_VER_MAJOR__ > 12)
KU_CUDA_CHECK(cudaGetDriverEntryPointByVersion(
"cuTensorMapEncodeTiled",
&pfn, 12000,
cudaEnableDefault,
&cuda_status));
#else
KU_CUDA_CHECK(cudaGetDriverEntryPoint(
"cuTensorMapEncodeTiled",
&pfn,
cudaEnableDefault,
&cuda_status));
#endif
if (cuda_status != cudaDriverEntryPointSuccess) {
KU_ASSERT(false, "Failed to load `cuTensorMapEncodeTiled`. cuda_status = %d", cuda_status);
}
return reinterpret_cast<decltype(&cuTensorMapEncodeTiled)>(pfn)(args...); \
};
CUtensorMap result;
CUresult ret_code = call_cuTensorMapEncodeTiled(
&result,
data_type,
dim,
global_ptr,
size.data(),
strides.data(),
box_size.data(),
element_strides.data(),
interleave_mode,
swizzle_mode,
l2_promotion,
oob_fill
);
if (ret_code != CUresult::CUDA_SUCCESS) {
auto print_vector = [&](auto t, const char* fmt, const char end='\n') {
for (auto elem : t) {
printf(fmt, elem);
}
printf("%c", end);
};
fprintf(stderr, "Failed to create tensormap\n");
fprintf(stderr, "Dim: %d\n", dim);
printf("size: "); print_vector(size, "%lu ");
printf("strides: "); print_vector(strides, "%lu ");
printf("box_size: "); print_vector(box_size, "%u ");
printf("element_strides: "); print_vector(element_strides, "%u ");
printf("global ptr: 0x%lx\n", (int64_t)global_ptr);
printf("data_type: %d\n", (int)data_type);
printf("swizzle_mode: %d\n", (int)swizzle_mode);
printf("l2_promotion: %d\n", (int)l2_promotion);
printf("interleave_mode: %d\n", (int)interleave_mode);
printf("oob_fill: %d\n", (int)oob_fill);
KU_ASSERT(false);
}
return result;
}
// Given strides (in number of elements), this function converts their datatype in uint64_t and then multiplies by elem_size
template<typename T>
static inline std::vector<uint64_t> make_stride_helper(const std::vector<T> &strides_in_elems, size_t elem_size) {
std::vector<uint64_t> res;
for (auto stride : strides_in_elems) {
res.push_back(((uint64_t)stride) * elem_size);
}
return res;
}
struct KernelLaunchConfig {
dim3 grid;
dim3 block;
size_t dynamic_smem;
cudaStream_t stream;
dim3 cluster{1, 1, 1};
bool use_pdl{false};
bool cooperative{false};
};
namespace detail {
template <class Arg>
void* kernel_arg_ptr(Arg&& arg) {
return const_cast<void*>(reinterpret_cast<const void*>(std::addressof(arg)));
}
} // namespace detail
template<typename KernelFunc, typename... Args>
void launch_kernel(const KernelLaunchConfig &cfg, KernelFunc kernel, Args&&... args) {
if (cfg.dynamic_smem > 0) {
KU_CUDA_CHECK(cudaFuncSetAttribute(
reinterpret_cast<const void*>(kernel),
cudaFuncAttributeMaxDynamicSharedMemorySize,
static_cast<int>(cfg.dynamic_smem)));
}
const bool is_cluster = !(cfg.cluster.x == 1 && cfg.cluster.y == 1 && cfg.cluster.z == 1);
const bool need_kernel_ex = is_cluster || cfg.use_pdl;
if (is_cluster && cfg.cooperative) {
KU_ASSERT(false, "cluster and cooperative launch are mutually exclusive");
}
if (cfg.cooperative) {
void* kernel_args[sizeof...(Args) > 0 ? sizeof...(Args) : 1] = {};
if constexpr (sizeof...(Args) > 0) {
size_t i = 0;
((kernel_args[i++] = detail::kernel_arg_ptr(std::forward<Args>(args))), ...);
}
KU_CUDA_CHECK(cudaLaunchCooperativeKernel(
kernel, cfg.grid, cfg.block,
sizeof...(Args) > 0 ? kernel_args : nullptr,
static_cast<unsigned int>(cfg.dynamic_smem), cfg.stream));
} else if (need_kernel_ex) {
if (is_cluster) {
KU_ASSERT(cfg.grid.x % cfg.cluster.x == 0 &&
cfg.grid.y % cfg.cluster.y == 0 &&
cfg.grid.z % cfg.cluster.z == 0);
if (cfg.cluster.x * cfg.cluster.y * cfg.cluster.z > 8) {
KU_CUDA_CHECK(cudaFuncSetAttribute(
reinterpret_cast<const void*>(kernel),
cudaFuncAttributeNonPortableClusterSizeAllowed, 1));
}
}
const unsigned int num_attrs = is_cluster ? 2 : 1;
cudaLaunchAttribute attrs[2];
if (is_cluster) {
attrs[0].id = cudaLaunchAttributeClusterDimension;
attrs[0].val.clusterDim = {cfg.cluster.x, cfg.cluster.y, cfg.cluster.z};
}
attrs[num_attrs - 1].id = cudaLaunchAttributeProgrammaticStreamSerialization;
attrs[num_attrs - 1].val.programmaticStreamSerializationAllowed = cfg.use_pdl ? 1 : 0;
cudaLaunchConfig_t config = {
{cfg.grid.x, cfg.grid.y, cfg.grid.z},
{cfg.block.x, cfg.block.y, cfg.block.z},
cfg.dynamic_smem,
cfg.stream,
attrs,
num_attrs
};
KU_CUDA_CHECK(cudaLaunchKernelEx(
&config, kernel, std::forward<Args>(args)...));
} else {
kernel<<<cfg.grid, cfg.block, cfg.dynamic_smem, cfg.stream>>>(
std::forward<Args>(args)...);
}
KU_CHECK_KERNEL_LAUNCH();
}
#endif // KERUTILS_IS_BUILD_ON_CUDA
} // namespace kerutils
@@ -0,0 +1,9 @@
#pragma once
#include "kerutils/common/common.h"
#include "host/host.h"
#ifdef KERUTILS_IS_BUILD_ON_CUDA
#include "device/device.cuh"
#endif
@@ -0,0 +1,21 @@
MIT License
Copyright (c) 2025 DeepSeek
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
@@ -0,0 +1,90 @@
#pragma once
#include <cstdint>
#include <cuda_bf16.h>
namespace topk_select_common {
__device__ __forceinline__
uint32_t bf16x2_to_u32(const nv_bfloat162 &v) {
static_assert(sizeof(nv_bfloat162) == sizeof(uint32_t));
return *reinterpret_cast<const uint32_t *>(&v);
}
__device__ __forceinline__
nv_bfloat162 u32_to_bf16x2(uint32_t u) {
static_assert(sizeof(nv_bfloat162) == sizeof(uint32_t));
nv_bfloat162 v;
*reinterpret_cast<uint32_t *>(&v) = u;
return v;
}
// Distort and un-distort: Map IEEE 754 floating-point total order to unsigned integer order.
// IEEE 754 bit representation (sign-magnitude with biased exponent and mantissa) does not
// preserve value order when interpreted as unsigned integer, and distort() fixes this:
// - For positive values (sign bit = 0): flip only the sign bit, mapping all positives to [0x80..., 0xFF...]
// - For negative values (sign bit = 1): flip all bits (~), inverting the magnitude order so that
// more negative (smaller value) => smaller uint
// After distortion, all negatives (0x00... ~ 0x7F...) < all positives (0x80... ~ 0xFF...) in
// unsigned integer comparison, which matches the floating-point total order.
//
// un_distort() is the exact inverse: flips the sign bit back for positives, flips all bits back for negatives.
template<typename UIntValueT>
__device__ __forceinline__
UIntValueT distort(const UIntValueT &x) {
static_assert(sizeof(UIntValueT) == 2 || sizeof(UIntValueT) == 4);
if constexpr (sizeof(UIntValueT) == 2) {
UIntValueT mask = (x&0x8000) ? 0xFFFF : 0x8000;
return x ^ mask;
} else {
// mask = (x >> 31) | 0x80000000 is 0x80000000 for positives and 0xFFFFFFFF for negatives
uint32_t t, d;
asm ("shr.s32 %0, %2, 31;\n"
"lop3.b32 %1, %2, %0, 0x80000000, 0x1e;\n"
: "=r"(t), "=r"(d)
: "r"((uint32_t)x));
return (UIntValueT)d;
}
}
// A SIMD-like version of `distort`, which uses more efficient instructions for calculation
template<typename UIntValueT>
__device__ __forceinline__
void distort_x2(UIntValueT result[2], const UIntValueT input[2]) {
static_assert(sizeof(UIntValueT) == 2 || sizeof(UIntValueT) == 4);
if constexpr (sizeof(UIntValueT) == 2) {
asm volatile (
"{"
".reg .u32 mask;"
"prmt.b32 mask, %1, 0, 0xbb99;" // mask[0:16] will be 0xFF if cur_value_packed[0]'s MSB is 1, 0x00 otherwise
"or.b32 mask, mask, 0x80008000;"
"xor.b32 %0, mask, %1;"
"}"
: "=r"(*(uint32_t*)(result))
: "r"(*(uint32_t*)(input))
);
} else {
result[0] = distort(input[0]);
result[1] = distort(input[1]);
}
}
template<typename UIntValueT>
__device__ __forceinline__
UIntValueT un_distort(const UIntValueT &x) {
static_assert(sizeof(UIntValueT) == 2 || sizeof(UIntValueT) == 4);
if constexpr (sizeof(UIntValueT) == 2) {
UIntValueT mask = (x&0x8000) ? 0x8000 : 0xFFFF;
return x ^ mask;
} else {
// mask = ~(x >> 31) | 0x80000000 is 0xFFFFFFFF for positives and 0x80000000 for negatives
uint32_t t, d;
asm ("shr.s32 %0, %2, 31;\n"
"lop3.b32 %1, %2, %0, 0x80000000, 0x4b;\n"
: "=r"(t), "=r"(d)
: "r"((uint32_t)x));
return (UIntValueT)d;
}
}
} // namespace topk_select_common
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,27 @@
#pragma once
#include <cstdint>
template<
typename ValueT_, typename OutIdxT_,
bool sorted_value_, bool sorted_index_, bool return_value_,
uint32_t max_topk_,
uint32_t num_threads_, uint32_t target_occupancy_,
uint32_t elements_per_round_, uint32_t reconstruct_threshold_, uint32_t tma_buffer_depth_, uint32_t elements_per_segment_ = 512,
uint32_t cluster_size_ = 1
>
struct TopkSelectConfig {
using ValueT = ValueT_;
using OutIdxT = OutIdxT_;
static constexpr bool sorted_value = sorted_value_;
static constexpr bool sorted_index = sorted_index_;
static constexpr bool return_value = return_value_;
static constexpr uint32_t max_topk = max_topk_;
static constexpr uint32_t num_threads = num_threads_;
static constexpr uint32_t target_occupancy = target_occupancy_; // Number of CTAs per SM
static constexpr uint32_t elements_per_round = elements_per_round_;
static constexpr uint32_t reconstruct_threshold = reconstruct_threshold_;
static constexpr uint32_t tma_buffer_depth = tma_buffer_depth_;
static constexpr uint32_t elements_per_segment = elements_per_segment_;
static constexpr uint32_t cluster_size = cluster_size_; // (1 = not a cluster variant)
};
@@ -0,0 +1,54 @@
#pragma once
#include <cstdint>
template<typename T>
__device__ __forceinline__ T warp_level_inclusive_prefix_sum(T x, uint32_t lane_idx) {
static_assert(sizeof(T) == 4);
#pragma unroll
for (uint32_t i = 1; i <= 16; i <<= 1) {
asm volatile (
"{\n"
".reg .pred p;\n"
".reg .b32 t;\n"
"shfl.sync.up.b32 t|p, %0, %1, 0, 0xffffffff;\n"
"@p add.u32 %0, %0, t;\n"
"}\n"
: "+r"(x)
: "r"(i)
);
}
return x;
}
template<typename T>
__device__ __forceinline__ T warp_level_exclusive_prefix_sum(T x, uint32_t lane_idx) {
T inclusive_prefix_sum = warp_level_inclusive_prefix_sum(x, lane_idx);
return inclusive_prefix_sum - x;
}
template<typename T>
__device__ __forceinline__ T warp_level_inclusive_suffix_sum(T x, uint32_t lane_idx) {
static_assert(sizeof(T) == 4);
#pragma unroll
for (uint32_t i = 1; i <= 16; i <<= 1) {
asm volatile (
"{\n"
".reg .pred p;\n"
".reg .b32 t;\n"
"shfl.sync.down.b32 t|p, %0, %1, 31, 0xffffffff;\n"
"@p add.u32 %0, %0, t;\n"
"}\n"
: "+r"(x)
: "r"(i)
);
}
return x;
}
template<typename T>
__device__ __forceinline__ T warp_level_exclusive_suffix_sum(T x, uint32_t lane_idx) {
T inclusive_suffix_sum = warp_level_inclusive_suffix_sum(x, lane_idx);
return inclusive_suffix_sum - x;
}
@@ -0,0 +1,147 @@
/*
TopK select kernel bf16 common variant.
Usage caveats:
- vocab_size must be < 2^23.
- Do NOT assume that the kernel will select a prefix index on ties.
*/
#pragma once
#include <cutlass/kernel_launch.h>
#include <cute/arch/copy_sm90_tma.hpp>
#include <kerutils/kerutils.cuh>
#include "structs.h"
#include "cuda_kernels/utils.cuh"
#include "cuda_kernels/common_parts.cuh"
namespace topk_select_bf16_normal {
template<typename Config>
class TopkSelectKernelBF16Normal : public topk_select_common::TopkSelectKernelBF16Base<Config> {
using BF16Base = topk_select_common::TopkSelectKernelBF16Base<Config>;
public:
using ValueT = typename BF16Base::ValueT;
using OutIdxT = typename BF16Base::OutIdxT;
using TmaParams = typename BF16Base::TmaParams;
using SharedMemoryPlanBase = typename BF16Base::SharedMemoryPlanBase;
using EpilogueT = typename BF16Base::EpilogueT;
using BF16Base::NUM_THREADS;
using BF16Base::MAX_TOPK;
using BF16Base::NUM_ELEMS_PER_SEG;
using BF16Base::NUM_ELEMS_PER_ROUND;
using BF16Base::NUM_SEGS_IN_INIT_WINDOW;
using BF16Base::NUM_INIT_ROUNDS_MAX;
using BF16Base::NUM_TAIL_ELEMS;
using BF16Base::NUM_TAIL_SEGS;
using BF16Base::NUM_SEGS_PER_ROUND;
static_assert(Config::target_occupancy >= 1 && Config::target_occupancy <= 8);
static_assert(MAX_TOPK == 512 || MAX_TOPK == 1024 || MAX_TOPK == 4096);
static constexpr bool HAS_PARTIAL_ROUNDS = NUM_SEGS_PER_ROUND > NUM_TAIL_SEGS;
struct SharedMemoryPlan : SharedMemoryPlanBase {};
static __device__ __forceinline__
void topk_select_kernel_devfunc(const TopkSelectArgs &args, const TmaParams &tma_params) {
uint32_t batch_idx = blockIdx.x;
uint32_t end_vocab_idx = args.end_ptr == nullptr ? args.vocab_size : __ldg(args.end_ptr + batch_idx);
extern __shared__ CUTE_ALIGNAS(1024) char wksp_buf[];
SharedMemoryPlan &smem = *reinterpret_cast<SharedMemoryPlan*>(wksp_buf);
uint32_t warp_idx = cutlass::canonical_warp_idx_sync();
uint32_t lane_idx = threadIdx.x % 32;
if (end_vocab_idx <= args.topk) {
EpilogueT::template topk_select_epilogue<true>(
(ValueT*)smem.surviving_topk_pairs[0],
(uint32_t*)(smem.surviving_topk_pairs[1]),
args,
batch_idx, end_vocab_idx, warp_idx,
*reinterpret_cast<typename EpilogueT::BlockRadixSortTempStorageT*>(smem.incoming_topk_pairs)
);
return;
}
uint32_t survivor_buf_idx = 0; // Current candidate buffer (A/B, swapped by reconstruct)
bool have_nan = false;
BF16Base::init_shared_memory(smem, warp_idx);
uint32_t num_input_segs = ku::ceil_div(end_vocab_idx, (uint32_t)NUM_ELEMS_PER_SEG);
uint32_t num_perm_segs = num_input_segs <= NUM_SEGS_IN_INIT_WINDOW ? 0u : (num_input_segs - 1) / NUM_TAIL_SEGS * NUM_TAIL_SEGS; // Skip permutation if the total number of segs is small enough
uint32_t num_perm_elems = num_perm_segs * NUM_ELEMS_PER_SEG;
uint32_t num_tail_elems_padded = num_perm_segs != 0 ? (uint32_t)NUM_TAIL_ELEMS : num_input_segs * NUM_ELEMS_PER_SEG;
uint32_t num_rounds = ku::ceil_div(num_tail_elems_padded + num_perm_elems, (uint32_t)NUM_ELEMS_PER_ROUND);
uint32_t num_init_rounds = min(num_rounds, (uint32_t)NUM_INIT_ROUNDS_MAX);
BF16Base::template scan_segs<false>(
tma_params, smem,
batch_idx, end_vocab_idx, args.topk,
warp_idx, lane_idx,
num_perm_segs,
0,
num_perm_segs,
num_tail_elems_padded,
end_vocab_idx - num_perm_elems,
survivor_buf_idx,
have_nan,
[&](uint32_t m) {
if constexpr (HAS_PARTIAL_ROUNDS) {
return (num_init_rounds + m) * NUM_SEGS_PER_ROUND + warp_idx < NUM_TAIL_SEGS + num_perm_segs;
} else {
return true;
}
}
);
// `__syncthreads_or` is also the barrier that makes the last `reconstruct`'s writes visible before
// `stage_output_and_epilogue` reads the survivor buffer.
have_nan = __syncthreads_or(have_nan) != 0;
if (have_nan) {
BF16Base::take_action_when_have_nan(args, batch_idx);
return;
}
BF16Base::template stage_output_and_epilogue<Config::sorted_index>(
smem, *reinterpret_cast<typename EpilogueT::BlockRadixSortTempStorageT*>(smem.incoming_topk_pairs), args,
batch_idx, end_vocab_idx, survivor_buf_idx, warp_idx, lane_idx);
}
};
template<typename Kernel>
__launch_bounds__(Kernel::NUM_THREADS, Kernel::TARGET_OCCUPANCY, 1)
__global__ void topk_kernel(__grid_constant__ const TopkSelectArgs args, __grid_constant__ const typename Kernel::TmaParams tma_params) {
Kernel::topk_select_kernel_devfunc(args, tma_params);
}
template<typename Config>
void run_topk_select_kernel(const TopkSelectArgs &args) {
KU_ASSERT(args.sorted_value == Config::sorted_value, "Dispatch failure");
KU_ASSERT(args.sorted_index == Config::sorted_index, "Dispatch failure");
KU_ASSERT(args.return_value == Config::return_value, "Dispatch failure");
static_assert(cute::is_same_v<typename Config::ValueT, nv_bfloat16>);
KU_ASSERT(args.vocab_size < MAX_VOCAB_SIZE, "`vocab_size` is too big");
using Kernel = TopkSelectKernelBF16Normal<Config>;
KU_ASSERT(args.topk <= Kernel::MAX_TOPK, "topk is too large. Maximum allowed: %d\n", Kernel::MAX_TOPK);
static_assert(INPUT_STRIDE_ALIGNMENT_REQUIREMENT % 16 == 0);
auto kernel = topk_kernel<Kernel>;
constexpr size_t smem_size = sizeof(typename Kernel::SharedMemoryPlan);
KU_ASSERT(smem_size * Kernel::TARGET_OCCUPANCY <= args.shared_memory_size_per_sm);
KU_CUDA_CHECK(cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, smem_size));
KU_ASSERT(args.stride_input_batch % 8 == 0, "stride_input_batch must be 16B-aligned");
typename Kernel::TmaParams tma_params = {Kernel::make_topk_tensor_map(args)};
ku::launch_kernel(ku::KernelLaunchConfig {
dim3(args.batch_size),
dim3(Kernel::NUM_THREADS),
smem_size,
args.stream
}, kernel, args, tma_params);
}
} // topk_select_bf16_normal
@@ -0,0 +1,305 @@
/*
TopK select kernel bf16 cluster variant. Use multiple CTAs to calculate one sequence to avoid wave quantization
Algorithm:
- Each CTA computes its local top-k pairs
- Each CTA sends its local top-k pairs to CTA0
- CTA0 selects top-k from those pairs
*/
#pragma once
#include <cutlass/kernel_launch.h>
#include <cute/arch/cluster_sm90.hpp>
#include <cute/arch/copy_sm90_tma.hpp>
#include <kerutils/kerutils.cuh>
#include "structs.h"
#include "cuda_kernels/utils.cuh"
#include "cuda_kernels/common_parts.cuh"
namespace topk_select_bf16_cluster {
CUTE_DEVICE
static void st_async_32b(uint32_t dst_addr, const uint32_t& data, transac_bar_t &mbar) {
uint32_t mbar_addr = cute::cast_smem_ptr_to_uint(&mbar);
asm volatile (
"st.async.weak.shared::cluster.mbarrier::complete_tx::bytes.s32 [%0], {%1}, [%2]; \n"
:
: "r"(dst_addr), "r"(data), "r"(mbar_addr)
);
}
template<typename Config>
class TopkSelectKernelBF16Cluster : public topk_select_common::TopkSelectKernelBF16Base<Config> {
using BF16Base = topk_select_common::TopkSelectKernelBF16Base<Config>;
public:
using ValueT = typename BF16Base::ValueT;
using OutIdxT = typename BF16Base::OutIdxT;
using TmaParams = typename BF16Base::TmaParams;
using SharedMemoryPlanBase = typename BF16Base::SharedMemoryPlanBase;
using EpilogueT = typename BF16Base::EpilogueT;
using BF16Base::NUM_THREADS;
using BF16Base::MAX_TOPK;
using BF16Base::NUM_EXTRA_SLOTS;
using BF16Base::NUM_ELEMS_PER_SEG;
using BF16Base::NUM_ELEMS_PER_ROUND;
using BF16Base::NUM_SEGS_IN_INIT_WINDOW;
using BF16Base::NUM_INIT_ROUNDS_MAX;
using BF16Base::NUM_TAIL_ELEMS;
using BF16Base::NUM_TAIL_SEGS;
using BF16Base::NUM_SEGS_PER_ROUND;
static_assert(Config::target_occupancy == 1);
static_assert(MAX_TOPK == 512 || MAX_TOPK == 1024);
static_assert(Config::cluster_size == 8 || Config::cluster_size == 16);
static_assert(NUM_ELEMS_PER_ROUND * sizeof(ValueT) % 1024 == 0);
static_assert((uint64_t)(MAX_VOCAB_SIZE / NUM_ELEMS_PER_SEG) * (MAX_VOCAB_SIZE / NUM_ELEMS_PER_SEG)
+ BF16Base::PERM_ADD_BASE <= 0xFFFFFFFFull);
static constexpr uint32_t CLUSTER_SIZE = Config::cluster_size;
static constexpr uint32_t NUM_GATHER_PAIRS = Config::cluster_size * MAX_TOPK;
static constexpr uint32_t NUM_GATHER_UNITS = NUM_GATHER_PAIRS / 2;
static constexpr uint32_t NUM_UINT32_GATHER_PER_THREAD = ((NUM_GATHER_UNITS + NUM_THREADS - 1) / NUM_THREADS) | 1u;
struct SharedMemoryPlanBF16Cluster : SharedMemoryPlanBase {
kerutils::transac_bar_t gather_val_bar;
kerutils::transac_bar_t gather_bar;
CUTE_ALIGNAS(16) uint32_t gathered_num_survivors[Config::cluster_size];
};
static_assert(NUM_GATHER_UNITS % NUM_THREADS == 0);
static_assert(NUM_GATHER_PAIRS <= 0xFFFF);
static_assert(2 * NUM_UINT32_GATHER_PER_THREAD <= 256);
static __device__ __forceinline__ void topk_select_kernel_devfunc(const TopkSelectArgs &args, const TmaParams &tma_params) {
uint32_t rank_in_cluster = blockIdx.x;
uint32_t batch_idx = blockIdx.y;
uint32_t end_vocab_idx = args.end_ptr == nullptr ? args.vocab_size : __ldg(args.end_ptr + batch_idx);
extern __shared__ CUTE_ALIGNAS(1024) char wksp_buf[];
SharedMemoryPlanBF16Cluster &smem = *reinterpret_cast<SharedMemoryPlanBF16Cluster*>(wksp_buf);
uint32_t warp_idx = cutlass::canonical_warp_idx_sync();
uint32_t lane_idx = threadIdx.x % 32;
if (end_vocab_idx <= args.topk) {
if (rank_in_cluster != 0) {
return;
}
EpilogueT::template topk_select_epilogue<true>(
(ValueT*)smem.surviving_topk_pairs[0],
(uint32_t*)(smem.surviving_topk_pairs[1]),
args,
batch_idx, end_vocab_idx, warp_idx,
*reinterpret_cast<typename EpilogueT::BlockRadixSortTempStorageT*>(smem.incoming_topk_pairs)
);
return;
}
uint32_t survivor_buf_idx = 0; // current candidate buffer (A/B, swapped by reconstruct)
BF16Base::init_shared_memory(smem, warp_idx, [&]{
smem.gather_val_bar.init(1);
smem.gather_bar.init(1);
});
// Still, divide the whole input into the "perm" part and the "tail" part
// The "tail" part contains the last 1 ~ NUM_TAIL_ELEMS elements and the "perm" part has the other elements.
uint32_t num_input_segs = ku::ceil_div(end_vocab_idx, (uint32_t)NUM_ELEMS_PER_SEG);
uint32_t num_perm_segs = num_input_segs <= NUM_SEGS_IN_INIT_WINDOW ? 0u : (num_input_segs - 1) / NUM_TAIL_SEGS * NUM_TAIL_SEGS;
uint32_t num_perm_elems = num_perm_segs * NUM_ELEMS_PER_SEG;
uint32_t num_tail_elems_padded = num_perm_segs != 0 ? (uint32_t)NUM_TAIL_ELEMS : num_input_segs * NUM_ELEMS_PER_SEG;
// The "tail" part is handled by CTA0 in the cluster
// "local" means the current CTA (in the cluster)
uint32_t num_local_tail_elems_padded = rank_in_cluster == 0 ? num_tail_elems_padded : 0u;
uint32_t num_local_tail_segs = num_local_tail_elems_padded / NUM_ELEMS_PER_SEG;
uint32_t num_local_tail_elems = rank_in_cluster == 0 ? end_vocab_idx - num_perm_elems : 0u;
// rank r takes the r-th contiguous interval of the row's visit order, with equal visit elements per rank
uint32_t num_total_elems_padded = num_tail_elems_padded + num_perm_elems;
// Rank r's slice of the padded row starts at r * num_total_elems_padded / cluster_size; the tail is
// rank 0's alone, so clamp that boundary into the perm part (element index within the perm part).
auto perm_boundary = [&](uint32_t rank) {
static_assert((uint64_t)Config::cluster_size * (MAX_VOCAB_SIZE + NUM_TAIL_ELEMS) <= 0xFFFFFFFFull);
uint32_t boundary = rank * num_total_elems_padded / Config::cluster_size;
return boundary > num_tail_elems_padded ? boundary - num_tail_elems_padded : 0u;
};
uint32_t local_start_seg_idx = perm_boundary(rank_in_cluster) / NUM_ELEMS_PER_SEG;
uint32_t local_end_seg_idx = perm_boundary(rank_in_cluster + 1) / NUM_ELEMS_PER_SEG;
uint32_t num_local_perm_segs = local_end_seg_idx - local_start_seg_idx;
uint32_t num_local_elems = num_local_tail_elems_padded + num_local_perm_segs * NUM_ELEMS_PER_SEG;
uint32_t num_local_rounds = ku::ceil_div(num_local_elems, (uint32_t)NUM_ELEMS_PER_ROUND);
uint32_t num_local_init_rounds = min(num_local_rounds, (uint32_t)NUM_INIT_ROUNDS_MAX);
uint32_t num_local_segs = num_local_tail_segs + num_local_perm_segs;
uint32_t used_segs = num_local_init_rounds * NUM_SEGS_PER_ROUND;
uint32_t rem_segs = num_local_segs > used_segs ? num_local_segs - used_segs : 0u;
uint32_t warp_rem_segs = warp_idx < rem_segs ? rem_segs - warp_idx : 0u;
uint32_t warp_round_limit = ku::ceil_div(warp_rem_segs, (uint32_t)NUM_SEGS_PER_ROUND);
// Each CTA gets its local top-k elements
bool nan_seen = false;
BF16Base::template scan_segs<true>(
tma_params, smem,
batch_idx, end_vocab_idx, args.topk, warp_idx, lane_idx,
num_perm_segs, local_start_seg_idx, num_local_perm_segs, num_local_tail_elems_padded, num_local_tail_elems,
survivor_buf_idx,
nan_seen,
[&](uint32_t m) { return m < warp_round_limit; }
);
// CTA-wide OR; this is also the barrier that makes the scan's writes to the survivor buffer visible to the gather below
nan_seen = __syncthreads_or(nan_seen) != 0;
// The cluster barrier must not complete before every CTA has finished its own scan: the gather
// below writes into CTA0's shared memory, so a CTA that finished early would otherwise clobber
// CTA0's `tma_load_buf` / `incoming_topk_pairs` while CTA0 is still scanning them.
ku::barrier_cluster_arrive_relaxed(); // Paired with the `barrier_cluster_wait_acquire` below
ku::barrier_cluster_wait_acquire();
static_assert(NUM_GATHER_UNITS * sizeof(uint32_t) <= sizeof(smem.incoming_topk_pairs));
void* val_dst = (void*)(int64_t)cute::set_block_rank(
cute::cast_smem_ptr_to_uint((uint32_t*)smem.incoming_topk_pairs + rank_in_cluster * (MAX_TOPK / 2)), 0);
static_assert(NUM_GATHER_PAIRS * sizeof(uint64_t) <= sizeof(SharedMemoryPlanBase::tma_load_buf));
void* dst = (void*)(int64_t)cute::set_block_rank(
cute::cast_smem_ptr_to_uint(reinterpret_cast<uint64_t*>(smem.tma_load_buf) + rank_in_cluster * MAX_TOPK), 0);
static_assert(MAX_TOPK*sizeof(ValueT) % 16 == 0);
constexpr uint32_t NUM_VAL_CHUNKS = MAX_TOPK * sizeof(ValueT) / 16; // 16B store
constexpr uint32_t NUM_VAL_CHUNKS_PER_THREAD = ku::ceil_div(NUM_VAL_CHUNKS, NUM_THREADS);
if (warp_idx == 0 && cute::elect_one_sync()) {
if (rank_in_cluster == 0) {
smem.gather_val_bar.arrive_and_expect_tx((NUM_VAL_CHUNKS * 16 + sizeof(uint32_t)) * CLUSTER_SIZE);
smem.gather_bar.arrive_and_expect_tx((MAX_TOPK * (uint32_t)sizeof(uint64_t)) * CLUSTER_SIZE);
}
st_async_32b(
cute::set_block_rank(cute::cast_smem_ptr_to_uint(smem.gathered_num_survivors + rank_in_cluster), 0),
(uint32_t)nan_seen,
smem.gather_val_bar
);
}
// Store values first... so that CTA0 can begin pivot selection earlier
CUTE_UNROLL
for (uint32_t i = 0; i < NUM_VAL_CHUNKS_PER_THREAD; ++i) {
uint32_t c = i * NUM_THREADS + threadIdx.x;
if constexpr (NUM_VAL_CHUNKS % NUM_THREADS != 0) {
if (c >= NUM_VAL_CHUNKS) break;
}
uint32_t vw[4];
CUTE_UNROLL
for (uint32_t j = 0; j < 4; ++j) {
uint32_t pair2[4];
topk_select_common::ld_shared<4>(pair2, (const uint32_t*)(smem.surviving_topk_pairs[survivor_buf_idx] + 8 * c + 2 * j));
vw[j] = __byte_perm(pair2[1], pair2[3], 0x5410);
}
ku::st_async((char*)val_dst + c * sizeof(uint4), make_uint4(vw[0], vw[1], vw[2], vw[3]),
smem.gather_val_bar);
}
// And then store those (value, index) pairs
static_assert(MAX_TOPK * sizeof(uint64_t) % 16 == 0);
constexpr uint32_t NUM_INDEX_VALUE_CHUNKS = MAX_TOPK * sizeof(uint64_t) / 16;
static_assert(NUM_INDEX_VALUE_CHUNKS % NUM_THREADS == 0);
CUTE_UNROLL
for (uint32_t i = 0; i < NUM_INDEX_VALUE_CHUNKS / NUM_THREADS; ++i) {
uint32_t s = 2 * (i * NUM_THREADS + threadIdx.x);
ulonglong2 pp = *reinterpret_cast<const ulonglong2*>(smem.surviving_topk_pairs[survivor_buf_idx] + s);
ku::st_async((char*)dst + s * sizeof(uint64_t), pp, smem.gather_bar);
}
if (rank_in_cluster != 0) {
return;
}
// The following code is CTA0 only
smem.gather_val_bar.wait(0);
BF16Base::clear_reconstruct_histograms(smem, threadIdx.x);
__syncthreads();
static_assert(Config::cluster_size <= 32);
uint32_t stored_num_survivors = lane_idx < Config::cluster_size ? smem.gathered_num_survivors[lane_idx] : 0u;
nan_seen |= stored_num_survivors;
uint32_t unit_base = threadIdx.x * NUM_UINT32_GATHER_PER_THREAD;
uint32_t num_my_units = unit_base < NUM_GATHER_UNITS ? min((uint32_t)NUM_UINT32_GATHER_PER_THREAD, NUM_GATHER_UNITS - unit_base) : 0u;
const uint32_t *gather_vals = (const uint32_t*)smem.incoming_topk_pairs;
nv_bfloat162 values[NUM_UINT32_GATHER_PER_THREAD];
CUTE_UNROLL
for (uint32_t i = 0; i < NUM_UINT32_GATHER_PER_THREAD; i++) {
if (i == num_my_units) break;
values[i] = topk_select_common::u32_to_bf16x2(gather_vals[unit_base + i]);
}
BF16Base::histogram_radix_msb(smem.reconstruct_bucket_counter[0], values, num_my_units);
__syncthreads();
auto [pivot_value_x2_bits, out_prefix, eq_quota, cnt_nan] =
BF16Base::template compute_pivot_and_quota<true>(args.topk, values, num_my_units, warp_idx, lane_idx, smem);
nan_seen |= cnt_nan != 0;
smem.gather_bar.wait(0);
__syncthreads();
uint32_t out_ptr = cute::cast_smem_ptr_to_uint(smem.surviving_topk_pairs[survivor_buf_idx ^ 1]) + out_prefix * (uint32_t)sizeof(uint64_t);
uint32_t gather_pairs_base = cute::cast_smem_ptr_to_uint(smem.tma_load_buf);
CUTE_UNROLL
for (uint32_t m = 0; m < NUM_UINT32_GATHER_PER_THREAD; m++) {
if (m == num_my_units) break;
BF16Base::template copy_selected_pairs_to_survivor<true>(out_ptr, eq_quota, values[m], pivot_value_x2_bits, gather_pairs_base + 2 * (unit_base + m) * (uint32_t)sizeof(uint64_t));
}
survivor_buf_idx ^= 1;
nan_seen = __syncthreads_or(nan_seen) != 0;
if (nan_seen) {
BF16Base::take_action_when_have_nan(args, batch_idx);
return;
}
BF16Base::template stage_output_and_epilogue<Config::sorted_index>(
smem, *reinterpret_cast<typename EpilogueT::BlockRadixSortTempStorageT*>(smem.incoming_topk_pairs), args,
batch_idx, end_vocab_idx, survivor_buf_idx, warp_idx, lane_idx);
}
};
template<typename Kernel>
__launch_bounds__(Kernel::NUM_THREADS, Kernel::TARGET_OCCUPANCY, Kernel::CLUSTER_SIZE)
__global__ void topk_kernel(__grid_constant__ const TopkSelectArgs args, __grid_constant__ const typename Kernel::TmaParams tma_params) {
Kernel::topk_select_kernel_devfunc(args, tma_params);
}
template<typename Config>
void run_topk_select_kernel(const TopkSelectArgs &args) {
KU_ASSERT(args.sorted_value == Config::sorted_value, "Dispatch failure");
KU_ASSERT(args.sorted_index == Config::sorted_index, "Dispatch failure");
KU_ASSERT(args.return_value == Config::return_value, "Dispatch failure");
static_assert(cute::is_same_v<typename Config::ValueT, nv_bfloat16>);
KU_ASSERT(args.vocab_size < MAX_VOCAB_SIZE, "`vocab_size` is too big");
using Kernel = TopkSelectKernelBF16Cluster<Config>;
KU_ASSERT(args.topk <= Kernel::MAX_TOPK, "topk is too large. Maximum allowed: %d\n", Kernel::MAX_TOPK);
static_assert(INPUT_STRIDE_ALIGNMENT_REQUIREMENT % 16 == 0);
auto kernel = topk_kernel<Kernel>;
constexpr size_t smem_size = sizeof(typename Kernel::SharedMemoryPlanBF16Cluster);
KU_ASSERT(smem_size * Kernel::TARGET_OCCUPANCY <= args.shared_memory_size_per_sm);
KU_CUDA_CHECK(cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, smem_size));
if constexpr (Kernel::CLUSTER_SIZE > 8) {
KU_CUDA_CHECK(cudaFuncSetAttribute(kernel, cudaFuncAttributeNonPortableClusterSizeAllowed, 1));
}
KU_ASSERT(args.stride_input_batch % 8 == 0, "stride_input_batch must be 16B-aligned");
typename Kernel::TmaParams tma_params = {Kernel::make_topk_tensor_map(args)};
ku::launch_kernel(ku::KernelLaunchConfig {
dim3(Kernel::CLUSTER_SIZE, args.batch_size),
dim3(Kernel::NUM_THREADS),
smem_size,
args.stream,
dim3(Kernel::CLUSTER_SIZE, 1, 1)
}, kernel, args, tma_params);
}
} // topk_select_bf16_cluster
@@ -0,0 +1,601 @@
/*
TopK select kernel fp32 variant.
Usage caveats:
- vocab_size must be < 2^23.
- Do NOT assume that the kernel will select a prefix index on ties.
*/
#pragma once
#include <cutlass/kernel_launch.h>
#include <cute/arch/copy_sm90_tma.hpp>
#include <kerutils/kerutils.cuh>
#include "structs.h"
#include "cuda_kernels/utils.cuh"
#include "cuda_kernels/common_parts.cuh"
namespace topk_select_fp32 {
template<typename Config>
class TopkSelectKernelFP32 : public topk_select_common::TopkSelectKernelBase<Config> {
using Base = topk_select_common::TopkSelectKernelBase<Config>;
public:
using ValueT = typename Base::ValueT;
using OutIdxT = typename Base::OutIdxT;
using TmaParams = typename Base::TmaParams;
using SharedMemoryPlanBase = typename Base::SharedMemoryPlanBase;
using EpilogueT = typename Base::EpilogueT;
using Base::NUM_THREADS;
using Base::NUM_WARPS;
using Base::MAX_TOPK;
using Base::NUM_ELEMS_PER_128b;
using Base::NUM_UINT32_PER_128b;
using Base::NUM_ELEMS_PER_ROUND;
using Base::NUM_ELEMS_PER_THREAD_PER_ROUND;
using Base::ELEMS_PER_THREAD_PER_ROUND_MASK;
using Base::NUM_128b_PER_THREAD_PER_ROUND;
using Base::NUM_ELEMS_PER_SEG;
using Base::NUM_TAIL_ELEMS;
using Base::NUM_TAIL_SEGS;
using Base::NUM_ELEMS_IN_INIT_WINDOW;
using Base::NUM_SEGS_IN_INIT_WINDOW;
using Base::NUM_PERM_SEGS_IN_INIT_WINDOW;
using Base::NUM_INIT_ROUNDS_MAX;
using Base::NUM_UINT32_IN_INIT_WINDOW_PER_THREAD;
using Base::NUM_128b_INIT_PER_THREAD;
using Base::NUM_SEGS_PER_ROUND;
using Base::NUM_SEGS_PER_ISSUE_WARP;
using Base::NUM_ISSUE_WARPS;
using Base::NUM_TMA_LOAD_BUFS;
using Base::TMA_PREFETCH_DEPTH;
using Base::RECONSTRUCT_THRESHOLD;
using Base::PERM_ADD_BASE;
using Base::PERM_MUL_PRIME;
using Base::PLACEHOLDER_PAIR;
using Base::NUM_RECONSTRUCT_BUCKETS;
static constexpr uint32_t PLACEHOLDER = 0xff800000; // -INF
static constexpr uint32_t NEG_INF_BITS = 0xFF800000;
static_assert(MAX_TOPK == 512 || MAX_TOPK == 1024 || MAX_TOPK == 4096);
static constexpr bool HAS_PARTIAL_ROUNDS = NUM_SEGS_PER_ROUND > NUM_TAIL_SEGS;
struct SharedMemoryPlanFP32 : SharedMemoryPlanBase {};
static __device__ __forceinline__
void append_pair_at(uint32_t &dst_ptr, uint32_t index, uint32_t val_bits) {
*(uint64_t*)__cvta_shared_to_generic(dst_ptr) = (uint64_t)val_bits << 32 | index;
dst_ptr = __float_as_uint(__uint_as_float(dst_ptr) + __uint_as_float(8u));
}
static __device__ __forceinline__ void topk_select_kernel_devfunc(const TopkSelectArgs &args, const TmaParams &tma_params) {
uint32_t batch_idx = blockIdx.x;
uint32_t end_vocab_idx = args.end_ptr == nullptr ? args.vocab_size : __ldg(args.end_ptr + batch_idx);
extern __shared__ CUTE_ALIGNAS(1024) char wksp_buf[];
SharedMemoryPlanFP32 &smem = *reinterpret_cast<SharedMemoryPlanFP32*>(wksp_buf);
uint32_t warp_idx = cutlass::canonical_warp_idx_sync();
uint32_t lane_idx = threadIdx.x % 32;
if (end_vocab_idx <= args.topk) {
EpilogueT::template topk_select_epilogue<true>(
(ValueT*)smem.surviving_topk_pairs[0],
(uint32_t*)(smem.surviving_topk_pairs[1]),
args,
batch_idx, end_vocab_idx, warp_idx,
*reinterpret_cast<typename EpilogueT::BlockRadixSortTempStorageT*>(smem.incoming_topk_pairs)
);
return;
}
uint32_t threshold_bits = NEG_INF_BITS;
uint32_t num_incomers = 0;
uint32_t survivor_buf_idx = 0; // current candidate buffer (A/B, swapped by reconstruct)
bool have_nan = false;
Base::init_shared_memory(smem, warp_idx);
uint32_t num_input_segs = ku::ceil_div(end_vocab_idx, (uint32_t)NUM_ELEMS_PER_SEG);
uint32_t num_perm_segs = num_input_segs <= NUM_SEGS_IN_INIT_WINDOW ? 0u : (num_input_segs - 1) / NUM_TAIL_SEGS * NUM_TAIL_SEGS;
uint32_t num_perm_elems = num_perm_segs * NUM_ELEMS_PER_SEG;
uint32_t num_local_tail_elems_padded = num_perm_segs != 0 ? (uint32_t)NUM_TAIL_ELEMS : num_input_segs * NUM_ELEMS_PER_SEG;
uint32_t num_rounds = ku::ceil_div(num_local_tail_elems_padded + num_perm_elems, (uint32_t)NUM_ELEMS_PER_ROUND);
uint32_t num_init_rounds = min(num_rounds, (uint32_t)NUM_INIT_ROUNDS_MAX);
ValueT *init_buf = (ValueT*)smem.incoming_topk_pairs; // aliased
uint32_t perm_len = max(num_perm_segs, 1u);
uint32_t perm_mul = PERM_MUL_PRIME % perm_len;
static_assert(NUM_SEGS_PER_ROUND == NUM_SEGS_PER_ISSUE_WARP * NUM_ISSUE_WARPS);
__shared__ bool should_select_whole_bucket_shared;
auto find_pivot_in_histogram = [&](uint32_t topk, const uint32_t *bucket_counter) {
bool should_select_whole_bucket =
Base::template find_pivot_in_histogram<true>(smem, bucket_counter, topk, lane_idx);
if (threadIdx.x == 0) {
should_select_whole_bucket_shared = should_select_whole_bucket;
}
};
// First radix pass: histogram the most significant byte of each distorted
// fp32 value. Used to locate the top-k bucket prefix.
uint32_t hist_base = cute::cast_smem_ptr_to_uint(smem.reconstruct_bucket_counter[0]);
// mad forms the bucket address and red.shared.add does the increment: no return value, one address add.
auto histogram_radix_msb_one = [&](uint32_t value_bits) {
uint32_t addr;
asm volatile ("mad.lo.u32 %0, %1, 4, %2; red.shared.add.u32 [%0], 1;"
: "=r"(addr)
: "r"(topk_select_common::distort(value_bits) >> 24), "r"(hist_base)
: "memory");
};
auto histogram_radix_msb = [&]<uint32_t N>(const uint32_t (&vals)[N], uint32_t m_end) {
CUTE_UNROLL
for (uint32_t i = 0; i < N; i++) {
if (i == m_end) break;
histogram_radix_msb_one(vals[i]);
}
};
struct PivotAndQuota {
uint32_t pivot_value_bits;
uint32_t start_pos_in_collector;
uint32_t eq_quota;
uint32_t cnt_nan;
};
auto compute_pivot_and_quota = [&](uint32_t topk, const auto &for_each_value) -> PivotAndQuota {
if (warp_idx == 0) {
find_pivot_in_histogram(topk, smem.reconstruct_bucket_counter[0]);
}
__syncthreads();
// `pivot_prefix` tracks the pivot's leading bits in the *undistorted* (raw) domain.
// Inside a round every element that survived the previous round must has the pivot's sign, so the distortion rule (to xor what) is the same.
// So we can optimize scanning & picking & distorting into "comparing the raw leading bits" and "xor-ing `neg_mask`"
uint32_t pivot_prefix_dist = smem.reconstruct_pivot_bucket; // distorted top 8 bits
bool pivot_negative = pivot_prefix_dist < 0x80u; // distorted >= 0x80 <=> positive
uint32_t neg_mask = pivot_negative ? 0xFFu : 0u;
uint32_t pivot_prefix = pivot_prefix_dist ^ (pivot_negative ? 0xFFu : 0x80u);
uint32_t num_elem_should_select_in_pivot_bucket = smem.reconstruct_num_should_select;
bool should_select_whole_bucket = should_select_whole_bucket_shared; // whether round 1 is already a clean boundary
uint32_t rounds_done = 1;
// Early exit: once topk lands exactly on a bucket's lower edge, the pivot is that bucket's floor.
CUTE_UNROLL
for (uint32_t r = 1; r < 4; ++r) {
if (should_select_whole_bucket) break;
uint32_t *dst = smem.reconstruct_bucket_counter[r & 1];
uint32_t *clr = smem.reconstruct_bucket_counter[(r & 1) ^ 1];
uint32_t dst_base = cute::cast_smem_ptr_to_uint(dst);
for_each_value([&](uint32_t value_bits) {
// Elements whose prefix differs from the pivot still add, but to this warp's own sink slot. This let PTXAS generate ATOMS.INC instructions which is faster
bool match = (value_bits >> (32 - 8 * r)) == pivot_prefix;
// bfe extracts the next raw byte in one instruction (ptxas otherwise emits shf + lop3)
uint32_t bucket_raw, bucket;
asm volatile ("bfe.u32 %0, %1, %2, 8;" : "=r"(bucket_raw) : "r"(value_bits), "r"(24 - 8 * r));
bucket = match ? (bucket_raw ^ neg_mask) : NUM_RECONSTRUCT_BUCKETS;
uint32_t addr;
asm volatile ("mad.lo.u32 %0, %1, 4, %2; red.shared.add.u32 [%0], 1;"
: "=r"(addr) : "r"(bucket), "r"(dst_base) : "memory");
});
constexpr uint32_t NUM_BUCKET_CLEAR_128b = NUM_RECONSTRUCT_BUCKETS / 4;
static_assert(NUM_THREADS >= NUM_BUCKET_CLEAR_128b);
if (threadIdx.x < NUM_BUCKET_CLEAR_128b) {
reinterpret_cast<__int128_t*>(clr)[threadIdx.x] = (__int128_t)0ull;
}
__syncthreads();
if (warp_idx == 0) {
find_pivot_in_histogram(num_elem_should_select_in_pivot_bucket, dst);
}
__syncthreads();
// The histogram is keyed by distorted bytes, so undo the sign map on the bucket id.
pivot_prefix = (pivot_prefix << 8) | (smem.reconstruct_pivot_bucket ^ neg_mask);
num_elem_should_select_in_pivot_bucket = smem.reconstruct_num_should_select;
should_select_whole_bucket = should_select_whole_bucket_shared;
rounds_done++;
}
// Take the lowest element in the selected bucket
// Be careful that `pivot` may be negative, and we may have to fill the lower bits with 1
uint32_t pivot_low_shift = (4 - rounds_done) * 8;
uint32_t pivot_value_bits = pivot_prefix << pivot_low_shift;
if (pivot_negative) {
pivot_value_bits |= (1u << pivot_low_shift) - 1u;
}
float pivot_value = __uint_as_float(pivot_value_bits);
// set.xx.f32.f32 yields 1.0/0.0, so the counts accumulate on the FP pipe instead of the
// (busiest) integer pipe; the counts stay exact because FP32 can represent every integer within 0 ~ 2**23
float gt_accum = 0.0f, eq_accum = 0.0f;
float nan_accum = 0.0f;
for_each_value([&](uint32_t value_bits) {
float v = __uint_as_float(value_bits);
asm volatile (
"{\n"
".reg .f32 g, e;\n"
"set.gt.f32.f32 g, %3, %4;\n"
"add.f32 %0, %0, g;\n"
"set.eq.f32.f32 e, %3, %4;\n"
"add.f32 %1, %1, e;\n"
"min.NaN.f32 %2, %2, %3;\n" // We use `min.NaN` for NaN detection
"}\n"
: "+f"(gt_accum), "+f"(eq_accum), "+f"(nan_accum)
: "f"(v), "f"(pivot_value));
});
uint32_t cnt_gt = (uint32_t)gt_accum;
uint32_t cnt_eq = (uint32_t)eq_accum;
float nan_flag;
asm ("set.nan.f32.f32 %0, %1, %1;" : "=f"(nan_flag) : "f"(nan_accum));
uint32_t cnt_nan = (uint32_t)nan_flag;
static_assert(NUM_WARPS <= NUM_RECONSTRUCT_BUCKETS);
uint32_t *eq_pass_warp_cnt = smem.reconstruct_bucket_counter[0];
auto eqgt = Base::compute_equal_quota_and_prefix(cnt_gt, cnt_eq, topk, warp_idx, lane_idx, eq_pass_warp_cnt);
return {pivot_value_bits, eqgt.start_pos_in_collector, eqgt.eq_quota, cnt_nan};
};
auto reconstruct = [&](uint32_t cur_extra_len) -> uint32_t {
uint32_t tid = threadIdx.x;
Base::clear_reconstruct_histograms(smem, tid);
uint32_t padded_extra_len = (cur_extra_len + 1u) & ~1u;
if (warp_idx == 0 && (cur_extra_len & 1u)) {
smem.incoming_topk_pairs[cur_extra_len] = PLACEHOLDER_PAIR;
}
__syncthreads();
uint32_t num_128b = (MAX_TOPK + padded_extra_len) / 2;
uint32_t num_128b_per_thread = ((num_128b + NUM_THREADS - 1) / NUM_THREADS) | 1u; // force odd => bank-conflict-free smem gathers
uint32_t b128_base = tid * num_128b_per_thread;
uint32_t num_my_128b = b128_base < num_128b ? min(num_128b_per_thread, num_128b - b128_base) : 0u;
auto b128_to_pair_addr = [&](uint32_t u) -> uint32_t {
// The first MAX_TOPK/2 b128 words live in the current candidate buffer, the rest in the incoming region
return u < MAX_TOPK / 2
? cute::cast_smem_ptr_to_uint(smem.surviving_topk_pairs[survivor_buf_idx] + 2 * u)
: cute::cast_smem_ptr_to_uint(smem.incoming_topk_pairs + (2 * u - MAX_TOPK));
};
// One b128 word is 16 bytes = 2 pairs {index0, value0, index1, value1};
constexpr uint32_t NUM_128b_MAX = (MAX_TOPK + RECONSTRUCT_THRESHOLD + NUM_ELEMS_PER_ROUND) / 2;
constexpr uint32_t NUM_128b_PER_THREAD = ((NUM_128b_MAX + NUM_THREADS - 1) / NUM_THREADS) | 1u; // | 1u since we may | 1u when generating `num_128b_per_thread`
auto for_each_value = [&](const auto &fn) {
CUTE_UNROLL
for (uint32_t i = 0; i < NUM_128b_PER_THREAD; i++) {
if (i == num_my_128b) break;
uint32_t pair2[4];
topk_select_common::ld_shared<4>(pair2, (const uint32_t*)b128_to_pair_addr(b128_base + i));
fn(pair2[1]);
fn(pair2[3]);
}
};
for_each_value(histogram_radix_msb_one);
__syncthreads();
uint32_t topk = args.topk;
auto [pivot_value_bits, start_pos_in_collector, eq_quota, cnt_nan] = compute_pivot_and_quota(topk, for_each_value);
// Every NaN the main loop collected is part of the buffer this census just walked (the hit test
// is `.gtu`, so NaN always becomes an incomer); the CTA-wide OR happens at the end.
have_nan |= cnt_nan != 0;
{
uint32_t out_ptr = cute::cast_smem_ptr_to_uint(smem.surviving_topk_pairs[survivor_buf_idx ^ 1]) + start_pos_in_collector * (uint32_t)sizeof(uint64_t);
float pivot_value = __uint_as_float(pivot_value_bits);
CUTE_UNROLL
for (uint32_t i = 0; i < NUM_128b_PER_THREAD; i++) {
if (i == num_my_128b) break;
uint32_t pair2[4];
topk_select_common::ld_shared<4>(pair2, (const uint32_t*)b128_to_pair_addr(b128_base + i));
float v0 = __uint_as_float(pair2[1]);
float v1 = __uint_as_float(pair2[3]);
bool t0 = v0 == pivot_value && eq_quota != 0;
eq_quota -= t0;
if (v0 > pivot_value || t0) { append_pair_at(out_ptr, pair2[0], pair2[1]); }
bool t1 = v1 == pivot_value && eq_quota != 0;
eq_quota -= t1;
if (v1 > pivot_value || t1) { append_pair_at(out_ptr, pair2[2], pair2[3]); }
}
}
survivor_buf_idx ^= 1;
return pivot_value_bits;
};
uint32_t tma_permuted_segment_stride = NUM_ISSUE_WARPS * perm_mul % perm_len;
uint32_t next_tma_permuted_segment = Base::get_permuted_seg_idx(warp_idx, perm_len, perm_mul);
auto issue_tma_loads_for_round = [&]<bool IS_INIT, bool HAVE_TAIL>(uint32_t idx) {
Base::template issue_tma_loads_for_round<IS_INIT, HAVE_TAIL>(
smem, tma_params.tensor_map, batch_idx, end_vocab_idx,
num_perm_segs, idx, next_tma_permuted_segment, tma_permuted_segment_stride, warp_idx);
};
if (cute::elect_one_sync()) {
issue_tma_loads_for_round.template operator()<true, true>(0);
CUTE_UNROLL
for (uint32_t i = 1; i < NUM_INIT_ROUNDS_MAX; i++) {
if (i < num_init_rounds) {
issue_tma_loads_for_round.template operator()<true, false>(i);
}
}
{
uint32_t num_main_prefetch = num_rounds - num_init_rounds;
CUTE_UNROLL
for (uint32_t i = 0; i < TMA_PREFETCH_DEPTH; i++) {
if (i < num_main_prefetch) {
issue_tma_loads_for_round.template operator()<false, false>(i);
}
}
}
}
{ // INIT
uint32_t num_elems_in_init_window_padded = num_local_tail_elems_padded + min(num_perm_segs, (uint32_t)NUM_PERM_SEGS_IN_INIT_WINDOW) * NUM_ELEMS_PER_SEG;
uint32_t num_128b_in_init_window_padded = num_elems_in_init_window_padded / NUM_ELEMS_PER_128b;
uint32_t cnt_floor = num_128b_in_init_window_padded / NUM_THREADS;
uint32_t cnt_rem = num_128b_in_init_window_padded % NUM_THREADS;
uint32_t my_elem_start_idx = (threadIdx.x * cnt_floor + min(threadIdx.x, cnt_rem)) * NUM_ELEMS_PER_128b;
uint32_t num_my_elems = NUM_UINT32_PER_128b * cnt_floor + (threadIdx.x < cnt_rem ? NUM_UINT32_PER_128b : 0u);
uint32_t num_local_tail_elems = end_vocab_idx - num_perm_elems;
if (num_perm_segs != 0) {
Base::fill_padded_tail_segments(init_buf, end_vocab_idx - num_perm_elems);
__syncthreads();
}
CUTE_UNROLL
for (uint32_t i = 0; i < NUM_INIT_ROUNDS_MAX; i++) {
if (i == num_init_rounds) break;
smem.init_full_bar[i].wait(0);
if (num_local_tail_elems % NUM_ELEMS_PER_SEG != 0 && i == num_local_tail_elems / NUM_ELEMS_PER_ROUND) {
uint32_t box_end = ku::ceil_div(num_local_tail_elems, (uint32_t)NUM_ELEMS_PER_SEG) * NUM_ELEMS_PER_SEG;
for (uint32_t e = num_local_tail_elems + threadIdx.x; e < box_end; e += NUM_THREADS) {
init_buf[Base::sw_elem(e)] = __uint_as_float(PLACEHOLDER);
}
__syncthreads();
}
uint32_t my_values[4 * NUM_128b_PER_THREAD_PER_ROUND];
uint32_t num_my_values = 0;
CUTE_UNROLL
for (uint32_t k = 0; k < NUM_128b_PER_THREAD_PER_ROUND; ++k) {
uint32_t u = threadIdx.x + (i * NUM_128b_PER_THREAD_PER_ROUND + k) * NUM_THREADS;
if (u < num_128b_in_init_window_padded) {
topk_select_common::ld_shared<4>(my_values + k * 4, (const uint32_t*)(init_buf + Base::sw_b128(u) * NUM_ELEMS_PER_128b));
num_my_values = 4 * (k + 1);
}
}
histogram_radix_msb(my_values, num_my_values);
}
uint32_t init_values[NUM_UINT32_IN_INIT_WINDOW_PER_THREAD];
CUTE_UNROLL
for (uint32_t j = 0; j < NUM_128b_INIT_PER_THREAD; ++j) {
if (j * NUM_UINT32_PER_128b == num_my_elems) break;
uint32_t u0 = my_elem_start_idx / NUM_ELEMS_PER_128b + j;
topk_select_common::ld_shared<4>(init_values + j * NUM_UINT32_PER_128b, (const uint32_t*)(init_buf + Base::sw_b128(u0) * NUM_ELEMS_PER_128b));
}
__syncthreads(); // publish the round-1 histogram; also the last read of init_buf
static_assert(NUM_ELEMS_IN_INIT_WINDOW <= 0xFFFF);
auto for_each_init_value = [&](const auto &fn) {
CUTE_UNROLL
for (uint32_t i = 0; i < NUM_UINT32_IN_INIT_WINDOW_PER_THREAD; i++) {
if (i == num_my_elems) break;
fn(init_values[i]);
}
};
auto [pivot_value_bits, start_pos_in_collector, eq_quota, cnt_nan] = compute_pivot_and_quota(args.topk, for_each_init_value);
// The init census walks the thread's whole slice of the window, i.e. every element of the init
// window exactly once, so NaNs inside the window are caught here.
have_nan |= cnt_nan != 0;
{
uint32_t out_ptr = cute::cast_smem_ptr_to_uint(smem.surviving_topk_pairs[0]) + start_pos_in_collector * (uint32_t)sizeof(uint64_t);
float pivot_value = __uint_as_float(pivot_value_bits);
uint32_t s0 = (max(my_elem_start_idx, num_local_tail_elems_padded) - num_local_tail_elems_padded) / NUM_ELEMS_PER_SEG;
uint32_t perm_state = Base::get_permuted_seg_idx(s0, perm_len, perm_mul);
float b128_base_f = 0.0f;
CUTE_UNROLL
for (uint32_t i = 0; i < NUM_UINT32_IN_INIT_WINDOW_PER_THREAD; i += 2) {
if (i == num_my_elems) break;
if (i % NUM_ELEMS_PER_128b == 0) {
uint32_t g_u = my_elem_start_idx + i; // fp32: word index == element index
if (g_u < num_local_tail_elems_padded) {
b128_base_f = __uint_as_float(num_perm_elems + g_u);
} else {
b128_base_f = __uint_as_float(perm_state * NUM_ELEMS_PER_SEG + g_u % NUM_ELEMS_PER_SEG);
if (g_u % NUM_ELEMS_PER_SEG == NUM_ELEMS_PER_SEG - NUM_ELEMS_PER_128b) {
Base::advance_perm_state(perm_state, perm_mul, perm_len);
}
}
}
// fp32-subnormal FP-pipe int-add (see file-head): index = base + element offset
uint32_t index0 = __float_as_uint(b128_base_f + __uint_as_float(i % NUM_ELEMS_PER_128b));
uint32_t index1 = __float_as_uint(b128_base_f + __uint_as_float(i % NUM_ELEMS_PER_128b + 1));
float v0 = __uint_as_float(init_values[i]);
float v1 = __uint_as_float(init_values[i + 1]);
bool t0 = v0 == pivot_value && eq_quota != 0;
eq_quota -= t0;
if (v0 > pivot_value || t0) { append_pair_at(out_ptr, index0, init_values[i]); }
// the quota is re-tested after element 0's decrement
bool t1 = v1 == pivot_value && eq_quota != 0;
eq_quota -= t1;
if (v1 > pivot_value || t1) { append_pair_at(out_ptr, index1, init_values[i + 1]); }
}
}
threshold_bits = pivot_value_bits;
__syncthreads();
}
uint32_t linear_segment_start =
num_init_rounds * NUM_SEGS_PER_ROUND
- NUM_TAIL_SEGS
+ warp_idx;
uint32_t current_permuted_segment = Base::get_permuted_seg_idx(linear_segment_start, perm_len, perm_mul);
uint32_t permuted_segment_stride_per_round = NUM_SEGS_PER_ROUND % perm_len * perm_mul % perm_len;
uint32_t num_main_rounds = num_rounds - num_init_rounds;
// Track the TMA buffer index and its phase incrementally instead of dividing every round.
uint32_t tma_buf_idx = 0;
uint32_t tma_buf_phase = 0;
for (uint32_t main_round_idx = 0; main_round_idx < num_main_rounds; ++main_round_idx) {
// This thread's contiguous chunk starts at logical_elem_offset in this round.
uint32_t logical_elem_offset = threadIdx.x * NUM_ELEMS_PER_THREAD_PER_ROUND;
// Offset inside the 512-element segment; used later to rebuild original indices.
uint32_t offset_in_segment = logical_elem_offset % NUM_ELEMS_PER_SEG;
// TMA stores the round data in a swizzled layout. swizzle_mask describes how the
// logical element offset is mapped to the actual shared-memory offset.
uint32_t swizzle_mask = Base::sw_msk(logical_elem_offset);
uint32_t chunk_swizzle_mask = swizzle_mask & ELEMS_PER_THREAD_PER_ROUND_MASK;
uint32_t smem_read_offset = logical_elem_offset ^ (swizzle_mask & ~ELEMS_PER_THREAD_PER_ROUND_MASK);
if (main_round_idx + TMA_PREFETCH_DEPTH < num_main_rounds) {
if (cute::elect_one_sync()) {
issue_tma_loads_for_round.template operator()<false, false>(main_round_idx + TMA_PREFETCH_DEPTH);
}
}
bool is_warp_active = true;
if constexpr (HAS_PARTIAL_ROUNDS) {
is_warp_active = (num_init_rounds + main_round_idx) * NUM_SEGS_PER_ROUND + warp_idx < NUM_TAIL_SEGS + num_perm_segs;
}
const ValueT *buf = smem.tma_load_buf[tma_buf_idx];
// One bit per element of this thread's slice, in element order.
// Hits are rare, so appending through the set bits of this mask is faster than testing + storing every element again.
static_assert(NUM_ELEMS_PER_THREAD_PER_ROUND <= 32); // hit_mask is a uint32_t
uint32_t hit_mask = 0;
if (is_warp_active) {
smem.tma_load_full_bar[tma_buf_idx].wait(tma_buf_phase);
ValueT values[NUM_ELEMS_PER_THREAD_PER_ROUND];
Base::template load_swizzled_slice<NUM_128b_PER_THREAD_PER_ROUND>(values, buf, smem_read_offset, chunk_swizzle_mask);
float threshold = __uint_as_float(threshold_bits);
CUTE_UNROLL
for (uint32_t j = 0; j < NUM_ELEMS_PER_THREAD_PER_ROUND; ++j) {
asm volatile (
"{\n"
".reg .pred p;\n"
"setp.gtu.f32 p, %1, %2;\n" // .gtu: NaN always counts as a hit, so every NaN reaches the incoming buffer and the census reports it
"@p or.b32 %0, %0, %3;\n"
"}\n"
: "+r"(hit_mask)
: "f"((float)values[j]), "f"(threshold), "r"(1u << j));
}
}
uint32_t num_new_incomers = __popc(hit_mask);
uint32_t warp_total_hits = __reduce_add_sync(0xFFFFFFFF, num_new_incomers);
if (lane_idx == 0) {
smem.warp_cnt[warp_idx] = warp_total_hits;
}
__syncthreads();
uint32_t seg_elem_base = current_permuted_segment * NUM_ELEMS_PER_SEG + offset_in_segment;
// Element e of this thread's slice lives at smem_read_offset + (e ^ chunk_swizzle_mask)
uint32_t elem_addr_base = cute::cast_smem_ptr_to_uint(buf + smem_read_offset);
// Start the first hit's smem load before the count exchange below, so that its latency
// (and the barrier wait) overlaps with them instead of delaying the first store.
uint32_t first_hit_e = hit_mask != 0 ? __ffs(hit_mask) - 1u : 0u;
uint32_t first_hit_val = 0;
if (is_warp_active && warp_total_hits != 0) {
asm volatile ("ld.shared.u32 %0, [%1];" : "=r"(first_hit_val) : "r"(elem_addr_base + ((first_hit_e ^ chunk_swizzle_mask) << 2)));
}
static_assert(NUM_WARPS <= 32);
uint32_t stored_warp_hits = lane_idx < NUM_WARPS ? smem.warp_cnt[lane_idx] : 0u;
uint32_t num_total_hits_in_this_round = __reduce_add_sync(0xFFFFFFFF, stored_warp_hits);
uint32_t dst_slot =
num_incomers +
__reduce_add_sync(0xFFFFFFFF, lane_idx < warp_idx ? stored_warp_hits : 0u) +
warp_level_exclusive_prefix_sum(num_new_incomers, lane_idx);
if (is_warp_active && warp_total_hits != 0) {
uint32_t dst_ptr = cute::cast_smem_ptr_to_uint(smem.incoming_topk_pairs) + dst_slot * (uint32_t)sizeof(uint64_t);
if (hit_mask != 0) {
asm volatile ("st.shared.v2.u32 [%0], {%1, %2};" :: "r"(dst_ptr), "r"(seg_elem_base + first_hit_e), "r"(first_hit_val) : "memory");
dst_ptr += (uint32_t)sizeof(uint64_t);
}
uint32_t mask = hit_mask & (hit_mask - 1u);
while (mask != 0) {
uint32_t e = __ffs(mask) - 1u;
mask &= mask - 1u;
uint32_t val_word;
asm volatile ("ld.shared.u32 %0, [%1];" : "=r"(val_word) : "r"(elem_addr_base + ((e ^ chunk_swizzle_mask) << 2)));
asm volatile ("st.shared.v2.u32 [%0], {%1, %2};" :: "r"(dst_ptr), "r"(seg_elem_base + e), "r"(val_word) : "memory");
dst_ptr += (uint32_t)sizeof(uint64_t);
}
}
Base::advance_perm_state(current_permuted_segment, permuted_segment_stride_per_round, perm_len);
num_incomers += num_total_hits_in_this_round;
__syncthreads();
if (++tma_buf_idx == NUM_TMA_LOAD_BUFS) {
tma_buf_idx = 0;
tma_buf_phase ^= 1u;
}
if (num_incomers >= RECONSTRUCT_THRESHOLD) {
threshold_bits = reconstruct(num_incomers);
num_incomers = 0;
}
}
if (num_incomers > 0) {
reconstruct(num_incomers);
num_incomers = 0;
}
have_nan = __syncthreads_or(have_nan) != 0;
if (have_nan) {
Base::take_action_when_have_nan(args, batch_idx);
return;
}
Base::template stage_output_and_epilogue<Config::sorted_index>(
smem, *reinterpret_cast<typename EpilogueT::BlockRadixSortTempStorageT*>(smem.incoming_topk_pairs), args,
batch_idx, end_vocab_idx, survivor_buf_idx, warp_idx, lane_idx);
}
};
template<typename Kernel>
__launch_bounds__(Kernel::NUM_THREADS, Kernel::TARGET_OCCUPANCY, 1)
__global__ void topk_kernel(__grid_constant__ const TopkSelectArgs args, __grid_constant__ const typename Kernel::TmaParams tma_params) {
Kernel::topk_select_kernel_devfunc(args, tma_params);
}
template<typename Config>
void run_topk_select_kernel(const TopkSelectArgs &args) {
KU_ASSERT(args.sorted_value == Config::sorted_value, "Dispatch failure");
KU_ASSERT(args.sorted_index == Config::sorted_index, "Dispatch failure");
KU_ASSERT(args.return_value == Config::return_value, "Dispatch failure");
static_assert(cute::is_same_v<typename Config::ValueT, float>);
KU_ASSERT(args.vocab_size < MAX_VOCAB_SIZE, "`vocab_size` is too big");
using Kernel = TopkSelectKernelFP32<Config>;
KU_ASSERT(args.topk <= Kernel::MAX_TOPK, "topk is too large. Maximum allowed: %d\n", Kernel::MAX_TOPK);
static_assert(INPUT_STRIDE_ALIGNMENT_REQUIREMENT % 16 == 0);
auto kernel = topk_kernel<Kernel>;
constexpr size_t smem_size = sizeof(typename Kernel::SharedMemoryPlanFP32);
KU_ASSERT(smem_size * Kernel::TARGET_OCCUPANCY <= args.shared_memory_size_per_sm);
KU_CUDA_CHECK(cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, smem_size));
KU_ASSERT(args.stride_input_batch % 4 == 0, "stride_input_batch must be 16B-aligned");
typename Kernel::TmaParams tma_params = {Kernel::make_topk_tensor_map(args)};
ku::launch_kernel(ku::KernelLaunchConfig {
dim3(args.batch_size),
dim3(Kernel::NUM_THREADS),
smem_size,
args.stream
}, kernel, args, tma_params);
}
} // topk_select_fp32
@@ -0,0 +1,39 @@
#pragma once
#include <cstdint>
#include <cuda_runtime_api.h>
static constexpr uint32_t INPUT_STRIDE_ALIGNMENT_REQUIREMENT = 1024; // In number of bytes
static constexpr uint32_t OUTPUT_STRIDE_ALIGNMENT_REQUIREMENT = 32; // In number of bytes
static constexpr uint32_t MAX_INT_ADDITION_RANGE_BY_FP32_SIMULATION = 1u << 23;
static constexpr uint32_t MAX_VOCAB_SIZE = 1u << 23;
static_assert(MAX_VOCAB_SIZE <= MAX_INT_ADDITION_RANGE_BY_FP32_SIMULATION);
struct TopkSelectArgs {
uint32_t batch_size;
uint32_t vocab_size;
uint32_t topk;
void* input;
void* output_value;
void* output_index;
int* begin_ptr;
int* end_ptr;
int* output_idx_offset;
// All strides are in number of elements, not bytes
uint64_t stride_input_batch;
uint64_t stride_output_value_batch;
uint64_t stride_output_index_batch;
bool sorted_value;
bool sorted_index;
bool return_value;
int idx_oob_fill_value;
float value_oob_fill_value;
bool abort_when_nan_found;
uint64_t shared_memory_size_per_sm;
cudaStream_t stream;
};
@@ -8,6 +8,8 @@
#include <sgl_kernel/utils.cuh>
#include <tvm/ffi/container/tensor.h>
#include <cstddef>
#include <cstdint>
#include <utility>
@@ -67,8 +69,13 @@ inline void* get_device_accessible_ptr(const tvm::ffi::TensorView& tensor) {
return device_ptr;
}
namespace details {
/**
* \brief Memo for a per-device driver query, keyed by device ordinal.
*
* \tparam T Cached value type.
* \tparam kDefault The "not queried yet" sentinel. A query that can legally
* return it would be re-run on every call, so pick one it cannot.
*/
template <typename T, T kDefault>
struct DeviceCacheMap {
public:
@@ -99,8 +106,6 @@ struct DeviceCacheMap {
T m_data[kNumStaticMaxDevice];
};
} // namespace details
// Return the maximum number of active blocks per SM for the given kernel
template <typename T>
inline auto get_blocks_per_sm(T&& kernel, int32_t block_dim, std::size_t dynamic_smem = 0) -> uint32_t {
@@ -111,7 +116,7 @@ inline auto get_blocks_per_sm(T&& kernel, int32_t block_dim, std::size_t dynamic
// Return the number of SMs for the given device
inline auto get_sm_count(int device_id, bool use_cache = true) -> uint32_t {
static details::DeviceCacheMap<uint32_t, 0> sm_count_cache;
static DeviceCacheMap<uint32_t, 0> sm_count_cache;
return sm_count_cache.get_cached(device_id, use_cache, [](int32_t device_id) {
int sm_count;
CHECK_CUDA(cudaDeviceGetAttribute(&sm_count, cudaDevAttrMultiProcessorCount, device_id));
@@ -121,7 +126,7 @@ inline auto get_sm_count(int device_id, bool use_cache = true) -> uint32_t {
// Return the Major compute capability for the given device
inline auto get_cc_major(int device_id, bool use_cache = true) -> int {
static details::DeviceCacheMap<int, -1> cc_major_cache;
static DeviceCacheMap<int, -1> cc_major_cache;
return cc_major_cache.get_cached(device_id, use_cache, [](int32_t device_id) {
int cc_major;
CHECK_CUDA(cudaDeviceGetAttribute(&cc_major, cudaDevAttrComputeCapabilityMajor, device_id));
@@ -131,7 +136,7 @@ inline auto get_cc_major(int device_id, bool use_cache = true) -> int {
// Return the Minor compute capability for the given device
inline auto get_cc_minor(int device_id, bool use_cache = true) -> int {
static details::DeviceCacheMap<int, -1> cc_minor_cache;
static DeviceCacheMap<int, -1> cc_minor_cache;
return cc_minor_cache.get_cached(device_id, use_cache, [](int32_t device_id) {
int cc_minor;
CHECK_CUDA(cudaDeviceGetAttribute(&cc_minor, cudaDevAttrComputeCapabilityMinor, device_id));
@@ -159,6 +164,26 @@ inline auto get_available_dynamic_smem_per_block(T&& kernel, int num_blocks, int
return smem_size;
}
/**
* \brief Shared memory one block may use on the given device, in bytes.
*
* \param device_id CUDA device ordinal.
* \param opt_in True for the larger limit a kernel reaches by setting
* `cudaFuncAttributeMaxDynamicSharedMemorySize`, false for the default
* cap that needs no opt-in.
* \param use_cache False re-queries the driver instead of reading the memo.
*/
inline auto get_max_smem_per_block(int device_id, bool opt_in = true, bool use_cache = true) -> std::size_t {
/// NOTE: we all know this is no larger than 1MB for all GPU, so use uint32_t
static DeviceCacheMap<uint32_t, 0> smem_cache[2];
const auto attr = opt_in ? cudaDevAttrMaxSharedMemoryPerBlockOptin : cudaDevAttrMaxSharedMemoryPerBlock;
return smem_cache[opt_in].get_cached(device_id, use_cache, [attr](int32_t device_id) {
int value = 0;
CHECK_CUDA(cudaDeviceGetAttribute(&value, attr, device_id));
return value;
});
}
} // namespace host::runtime
} // namespace sglang
@@ -7,6 +7,7 @@ from sglang.kernels.jit.utils.arch import (
override_jit_cuda_arch,
)
from sglang.kernels.jit.utils.common import (
aligned_new_empty,
cache_once,
empty_sentinel,
get_ci_test_range,
@@ -23,6 +24,7 @@ from sglang.kernels.jit.utils.compile import (
)
__all__ = [
"aligned_new_empty",
"empty_sentinel",
"should_run_full_tests",
"get_ci_test_range",
+26 -1
View File
@@ -3,7 +3,7 @@
from __future__ import annotations
import functools
from typing import Any, Callable, Dict, List, TypeVar
from typing import Any, Callable, Dict, List, Sequence, TypeVar
import torch
@@ -86,3 +86,28 @@ def lazy_register_class(name: str, init_fn: Callable[[], None]) -> Callable[[T],
return cls
return decorator
def aligned_new_empty(
shape: Sequence[int],
dtype: torch.dtype,
device: torch.device,
*,
alignment: int,
) -> torch.Tensor:
"""An uninitialized tensor whose row stride is a multiple of `alignment` bytes.
The last dimension is padded up and then sliced back off, so the result has
the requested shape and a non-contiguous stride whenever padding was needed.
Callers are kernels that state an alignment contract on a row address, such
as a TMA descriptor or a vectorized load.
"""
assert len(shape) > 1
assert alignment % dtype.itemsize == 0, (
"alignment must be a whole number of elements"
)
step = alignment // dtype.itemsize
last_dim = shape[-1]
padded_last_dim = (last_dim + step - 1) // step * step
aligned_shape = (*shape[:-1], padded_last_dim)
return torch.empty(aligned_shape, dtype=dtype, device=device)[..., :last_dim]
+320
View File
@@ -0,0 +1,320 @@
from __future__ import annotations
from typing import TYPE_CHECKING, Dict, NamedTuple, Optional, Tuple
import torch
from sglang.kernels.jit.utils import (
KERNEL_PATH,
aligned_new_empty,
cache_once,
load_jit,
make_cpp_args,
)
if TYPE_CHECKING:
from tvm_ffi.module import Module
# Must match INPUT_/OUTPUT_STRIDE_ALIGNMENT_REQUIREMENT in vendor/structs.h; the
# kernel's TMA descriptor depends on them and `entry.cuh` rejects a row that
# misses one.
_INPUT_STRIDE_ALIGNMENT_BYTES = 1024
_OUTPUT_STRIDE_ALIGNMENT_BYTES = 32
_CLUSTER_MAX_TOPK = 1024
_CLUSTER_MAX_BATCH_SIZE = 6
_SUPPORTED_CAPABILITIES = ((9, 0), (10, 0), (10, 3))
class _Config(NamedTuple):
max_topk: int
num_threads: int
occupancy: int
elements_per_round: int
reconstruct_threshold: int
tma_buffer_depth: int
cluster_size: int = 1
def to_cpp_class_name(
self,
value_dtype: torch.dtype,
index_dtype: torch.dtype,
sorted_value: bool,
sorted_index: bool,
return_value: bool,
) -> str:
template_args = make_cpp_args(
value_dtype,
index_dtype,
sorted_value,
sorted_index,
return_value,
self.max_topk,
self.num_threads,
self.occupancy,
self.elements_per_round,
self.reconstruct_threshold,
self.tma_buffer_depth,
512, # elements_per_segment, fixed upstream
self.cluster_size,
)
return f"TopkSelectConfig<{template_args}>"
# Per (value dtype, top-k bucket): the tuning for a single-wave batch and, where
# the bucket splits on wave count, the tuning for anything larger. A single
# entry means no split, and then only one kernel is instantiated.
_NORMAL_CONFIGS: Dict[Tuple[torch.dtype, int], Tuple[_Config, ...]] = {
(torch.bfloat16, 512): (
_Config(512, 512, 1, 8192, 4096, 5),
_Config(512, 256, 2, 4096, 4096, 4),
),
(torch.bfloat16, 1024): (
_Config(1024, 512, 1, 8192, 4096, 5),
_Config(1024, 256, 2, 4096, 4096, 3),
),
(torch.bfloat16, 4096): (_Config(4096, 512, 1, 8192, 4096, 3),),
(torch.float32, 512): (_Config(512, 512, 1, 8192, 4096, 3),),
(torch.float32, 1024): (_Config(1024, 512, 1, 8192, 4096, 3),),
(torch.float32, 4096): (_Config(4096, 256, 1, 4096, 4096, 3),),
}
@cache_once
def _cluster_tuning(device_capability: Tuple[int, int]) -> Tuple[int, int]:
# Return (cluster_size, min_vocab_size)
major = device_capability[0]
return (8, 128 * 1024) if major == 9 else (16, 512 * 1024)
@cache_once
def _jit_deep_select_module(
value_dtype: torch.dtype,
index_dtype: torch.dtype,
sorted_value: bool,
sorted_index: bool,
return_value: bool,
max_topk: int,
cluster_size: int,
) -> Module:
assert value_dtype in (torch.bfloat16, torch.float32)
assert index_dtype in (torch.int32, torch.int64)
if sorted_value:
assert value_dtype == torch.float32
assert return_value and not sorted_index
if cluster_size == 1:
configs = _NORMAL_CONFIGS[(value_dtype, max_topk)]
host_dispatch = "TopkNormal"
else:
# Upstream tunes the cluster kernel for bf16 at max_topk 1024 alone, so
# the 512 and 1024 buckets share this module.
assert value_dtype == torch.bfloat16
assert max_topk == _CLUSTER_MAX_TOPK
configs = [_Config(max_topk, 256, 1, 4096, 4096, 16, cluster_size)]
host_dispatch = "TopkCluster"
classes = [
config.to_cpp_class_name(
value_dtype=value_dtype,
index_dtype=index_dtype,
sorted_value=sorted_value,
sorted_index=sorted_index,
return_value=return_value,
)
for config in configs
]
classes = make_cpp_args(*classes)
root = (KERNEL_PATH / "csrc" / "deepselect" / "vendor").resolve()
return load_jit(
"deep_select_topk",
# cache only distinct key for better readability
*make_cpp_args(
value_dtype,
index_dtype,
sorted_value,
sorted_index,
return_value,
max_topk,
cluster_size,
),
cuda_files=["deepselect/entry.cuh"],
cuda_wrappers=[("topk", f"deepselect::{host_dispatch}<{classes}>::topk")],
extra_include_paths=[
str(root),
str(root / "3rdparty" / "kerutils" / "include"),
],
extra_dependencies=["cutlass"],
extra_cuda_cflags=[
"--expt-extended-lambda",
"--use_fast_math",
"--ftz=false",
"-Xptxas=--register-usage-level=10",
],
)
def _get_max_topk_bucket(topk: int, use_cluster: bool) -> int:
if use_cluster:
return _CLUSTER_MAX_TOPK
return 512 if topk <= 512 else 1024 if topk <= 1024 else 4096
def get_input_stride_alignment_bytes() -> int:
return _INPUT_STRIDE_ALIGNMENT_BYTES
def get_stride_requirement() -> Tuple[int, int]:
"""Return the input and output row-stride requirements in bytes."""
return _INPUT_STRIDE_ALIGNMENT_BYTES, _OUTPUT_STRIDE_ALIGNMENT_BYTES
def get_deepselect_supported_architectures() -> Tuple[int, ...]:
"""Return the CUDA compute capabilities supported by the JIT kernel."""
return tuple(major * 10 + minor for major, minor in _SUPPORTED_CAPABILITIES)
def is_deepselect_supported(device=None) -> bool:
"""Return whether DeepSelect JIT supports a CUDA device."""
if torch.version.cuda is None or not torch.cuda.is_available():
return False
try:
normalized_device = (
torch.device("cuda", device)
if isinstance(device, int)
else (
torch.device("cuda", torch.cuda.current_device())
if device is None
else torch.device(device)
)
)
if normalized_device.type != "cuda":
return False
return (
torch.cuda.get_device_capability(normalized_device)
in _SUPPORTED_CAPABILITIES
)
except (RuntimeError, TypeError, ValueError):
return False
def _needs_output_staging(output: torch.Tensor, topk: int) -> bool:
return (
output.data_ptr() % _OUTPUT_STRIDE_ALIGNMENT_BYTES != 0
or topk * output.element_size() % _OUTPUT_STRIDE_ALIGNMENT_BYTES != 0
)
def topk(
input: torch.Tensor,
topk: int,
sorted: bool = False,
begin: Optional[torch.Tensor] = None,
end: Optional[torch.Tensor] = None,
indices_type: torch.dtype = torch.int64,
sorted_index: bool = False,
hint: Optional[torch.Tensor] = None,
output_idx: Optional[torch.Tensor] = None,
output_idx_offset: Optional[torch.Tensor] = None,
idx_oob_fill_value: int = 2147483647,
value_oob_fill_value: float = float("-inf"),
return_value: bool = True,
abort_when_nan_found: bool = True,
) -> Tuple[Optional[torch.Tensor], torch.Tensor]:
"""Select the largest ``topk`` values of every row of ``input``.
This follows the public DeepSelect interface. ``end`` is the per-row
exclusive valid length; ``begin`` and ``hint`` are reserved but unsupported.
Returns ``(values, indices)``; ``values`` is None when ``return_value`` is
False, which skips writing them and is about 10% faster.
"""
if hint is not None:
raise ValueError("hint is not supported currently")
if input.dtype not in (torch.bfloat16, torch.float32):
raise RuntimeError("input dtype must be bfloat16 or float32")
if indices_type not in (torch.int32, torch.int64):
raise RuntimeError("indices_type must be int32 or int64")
if output_idx is not None and output_idx.dtype != indices_type:
raise ValueError("output_idx dtype must match indices_type")
if input.device.type != "cuda":
raise RuntimeError("input must be a CUDA tensor")
if input.ndim != 2:
raise RuntimeError("input must be a 2D tensor")
if not 0 < topk <= 4096:
raise RuntimeError(f"topk must be in [1, 4096], got {topk}")
if sorted and not return_value:
raise RuntimeError("return_value must be enabled when sorted is True")
if sorted and sorted_index:
raise RuntimeError("sorted and sorted_index cannot both be True")
if sorted and input.dtype is torch.bfloat16:
raise RuntimeError("sorted is only supported for float32 input")
rows, vocab_size = input.shape
device_capability = torch.cuda.get_device_capability(input.device)
if device_capability not in _SUPPORTED_CAPABILITIES:
major, minor = device_capability
raise RuntimeError(f"DeepSelect does not support SM{major}{minor}")
cluster_size, cluster_min_vocab_size = _cluster_tuning(device_capability)
use_cluster = (
input.dtype is torch.bfloat16
and rows <= _CLUSTER_MAX_BATCH_SIZE
and vocab_size >= cluster_min_vocab_size
and topk <= _CLUSTER_MAX_TOPK
)
values = None
if return_value:
values = aligned_new_empty(
(rows, topk),
input.dtype,
input.device,
alignment=_OUTPUT_STRIDE_ALIGNMENT_BYTES,
)
if output_idx is None:
output_idx = aligned_new_empty(
(rows, topk),
indices_type,
input.device,
alignment=_OUTPUT_STRIDE_ALIGNMENT_BYTES,
)
kernel_output_idx = output_idx
else:
kernel_output_idx = (
aligned_new_empty(
(rows, topk),
indices_type,
input.device,
alignment=_OUTPUT_STRIDE_ALIGNMENT_BYTES,
)
if _needs_output_staging(output_idx, topk)
else output_idx
)
module = _jit_deep_select_module(
input.dtype,
indices_type,
sorted,
sorted_index,
return_value,
_get_max_topk_bucket(topk, use_cluster),
cluster_size if use_cluster else 1,
)
input_storage_bytes = (
input.untyped_storage().nbytes() - input.storage_offset() * input.element_size()
)
module.topk(
input,
values,
output_idx,
kernel_output_idx,
begin,
end,
output_idx_offset,
topk,
input_storage_bytes,
idx_oob_fill_value,
value_oob_fill_value,
abort_when_nan_found,
)
if kernel_output_idx is not output_idx:
output_idx.copy_(kernel_output_idx)
return values, output_idx
deepselect_topk = topk