[DeepSeek V4.1] Add DeepSelect JIT kernel. (#40556)
Co-authored-by: DarkSharpness <2040703891@qq.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
DarkSharpness
Claude Opus 5
parent
a5c8afab48
commit
7bc988446d
@@ -0,0 +1,59 @@
|
||||
# DeepSelect top-k
|
||||
|
||||
`vendor/` is [deepseek-ai/DeepSelect](https://github.com/deepseek-ai/DeepSelect)
|
||||
(commit `382d62a`, plus the SM90 cluster variant from upstream PR #14), stripped
|
||||
to the kernel implementation. MIT licensed; see `vendor/LICENSE`.
|
||||
|
||||
`entry.cuh` is the SGLang side: tensor validation, the `TopkSelectArgs` block,
|
||||
and the two host entry points the JIT exports. `python/sglang/kernels/ops/deep_select.py`
|
||||
is the wrapper that compiles and calls them.
|
||||
|
||||
## What is vendored
|
||||
|
||||
| Path under `vendor/` | What it is |
|
||||
| --- | --- |
|
||||
| `structs.h` | `TopkSelectArgs` -- the launcher's plain-C argument block, plus the stride-alignment constants |
|
||||
| `cuda_kernels/config.h` | `TopkSelectConfig<...>` -- every compile-time knob of a kernel instance |
|
||||
| `cuda_kernels/{utils,bit_utils}.cuh` | small device helpers |
|
||||
| `cuda_kernels/common_parts.cuh` | the shared kernel body (TMA pipeline, radix pass, epilogue) |
|
||||
| `cuda_kernels/v3/topk_select.cuh` | bf16, one CTA per row |
|
||||
| `cuda_kernels/v3_cluster/topk_select.cuh` | bf16, one cluster per row (8 CTAs on SM90, 16 on SM100/SM103) |
|
||||
| `cuda_kernels/v3_fp32/topk_select.cuh` | fp32 |
|
||||
| `3rdparty/kerutils/` | upstream's header-only helper library, subset |
|
||||
|
||||
Each `topk_select.cuh` defines one kernel plus its launcher,
|
||||
`run_topk_select_kernel<Config>(const TopkSelectArgs&)`, in its own namespace
|
||||
(`topk_select_bf16_normal`, `topk_select_bf16_cluster`, `topk_select_fp32`).
|
||||
|
||||
Dropped from the vendored tree, because a JIT build does not need them:
|
||||
|
||||
- `cuda_kernels/*/instantiations/**` -- 86 explicit instantiation `.cu` files.
|
||||
They exist so an AOT build can compile every configuration into one shared
|
||||
object. `entry.cuh` instantiates the one or two a module can reach.
|
||||
- `api.cpp` -- the torch/pybind entry point and the runtime dispatch that
|
||||
picked a configuration from `(dtype, topk, vocab_size, batch_size, arch)`.
|
||||
That decision is now split between the Python module factory and `entry.cuh`.
|
||||
- `dispatch_utils.h` -- `BOOL_SWITCH` / `INTEGER_TYPE_SWITCH`, which include
|
||||
`torch/extension.h` and exist only to expand that dispatch.
|
||||
- `cuda_kernels/*/topk_select.h` -- forward declarations of
|
||||
`run_topk_select_kernel`, needed only to split the instantiations into
|
||||
separate translation units.
|
||||
|
||||
## Build inputs
|
||||
|
||||
Include paths: `vendor/` (the `#include "cuda_kernels/..."` lines resolve
|
||||
against it) and `vendor/3rdparty/kerutils/include`. CUTLASS supplies
|
||||
`cute/arch/copy_sm90_tma.hpp` and `cutlass/arch/barrier.h`, and is a registered
|
||||
JIT dependency (`extra_dependencies=["cutlass"]`).
|
||||
|
||||
Beyond the JIT defaults the kernels need `--expt-extended-lambda` and
|
||||
`--ftz=false`; the latter is load-bearing and must follow `--use_fast_math`,
|
||||
because the radix pass simulates integer counters with fp32 addition and needs
|
||||
subnormals.
|
||||
|
||||
## Re-syncing with upstream
|
||||
|
||||
Copy the files listed above into `vendor/`. They are unmodified apart from
|
||||
trailing whitespace and the removed `#include "topk_select.h"` line at the top
|
||||
of each `topk_select.cuh`. `vendor/.clang-format` disables formatting so a
|
||||
re-sync stays a plain copy.
|
||||
@@ -0,0 +1,294 @@
|
||||
#pragma once
|
||||
|
||||
// The vendored DeepSelect sources declare everything at global scope, so they
|
||||
// are included before `namespace sglang` is opened.
|
||||
#include <sgl_kernel/tensor.h>
|
||||
#include <sgl_kernel/utils.h>
|
||||
|
||||
#include <sgl_kernel/runtime.cuh>
|
||||
#include <sgl_kernel/utils.cuh>
|
||||
|
||||
#include <tvm/ffi/container/tensor.h>
|
||||
#include <tvm/ffi/optional.h>
|
||||
|
||||
#include "vendor/cuda_kernels/v3/topk_select.cuh"
|
||||
#include "vendor/cuda_kernels/v3_cluster/topk_select.cuh"
|
||||
#include "vendor/cuda_kernels/v3_fp32/topk_select.cuh"
|
||||
#include <cstdint>
|
||||
#include <limits>
|
||||
#include <type_traits>
|
||||
|
||||
namespace sglang {
|
||||
|
||||
/**
|
||||
* \brief Host entry points for the vendored DeepSelect top-k kernels.
|
||||
*
|
||||
* Upstream resolves a `TopkSelectConfig` from the call at runtime, expanding
|
||||
* every boolean and index-type axis through nested `BOOL_SWITCH` macros. Here
|
||||
* those axes -- value dtype, index dtype, `sorted_value`, `sorted_index`,
|
||||
* `return_value`, the top-k bucket and the cluster size -- are template
|
||||
* arguments picked by the Python module factory, which leaves exactly one
|
||||
* decision for C++: which of a bucket's two tunings the launch shape wants.
|
||||
*/
|
||||
namespace deepselect {
|
||||
|
||||
namespace details {
|
||||
|
||||
/// The two families of one-CTA-per-row kernels live in separate namespaces.
|
||||
template <typename Config>
|
||||
inline auto run_normal(const TopkSelectArgs& args) -> void {
|
||||
if constexpr (std::is_same_v<typename Config::ValueT, float>) {
|
||||
topk_select_fp32::run_topk_select_kernel<Config>(args);
|
||||
} else {
|
||||
topk_select_bf16_normal::run_topk_select_kernel<Config>(args);
|
||||
}
|
||||
}
|
||||
|
||||
/// Validated launch arguments plus the device they were validated against.
|
||||
struct LaunchPlan {
|
||||
TopkSelectArgs args;
|
||||
DLDevice device;
|
||||
};
|
||||
|
||||
/**
|
||||
* \brief Validate the tensors and build the argument block for one launch.
|
||||
*
|
||||
* \tparam Config The specialisation the caller is about to run. Its
|
||||
* `sorted_value` / `sorted_index` / `return_value` become the runtime
|
||||
* flags, which the kernel launcher then asserts back against them.
|
||||
*/
|
||||
template <typename Config>
|
||||
auto make_plan(
|
||||
tvm::ffi::TensorView input,
|
||||
tvm::ffi::Optional<tvm::ffi::TensorView> output_value,
|
||||
tvm::ffi::TensorView output_index,
|
||||
tvm::ffi::TensorView kernel_output_index,
|
||||
tvm::ffi::Optional<tvm::ffi::TensorView> begin,
|
||||
tvm::ffi::Optional<tvm::ffi::TensorView> end,
|
||||
tvm::ffi::Optional<tvm::ffi::TensorView> output_index_offset,
|
||||
int64_t topk,
|
||||
int64_t input_storage_bytes,
|
||||
int64_t idx_oob_fill_value,
|
||||
double value_oob_fill_value,
|
||||
bool abort_when_nan_found) -> LaunchPlan {
|
||||
using namespace host;
|
||||
using ValueT = typename Config::ValueT;
|
||||
using OutIdxT = typename Config::OutIdxT;
|
||||
|
||||
auto batch = SymbolicSize{"batch_size"};
|
||||
auto vocab = SymbolicSize{"vocab_size"};
|
||||
auto input_stride = SymbolicSize{"input_row_stride"};
|
||||
auto index_stride = SymbolicSize{"index_row_stride"};
|
||||
auto kernel_index_stride = SymbolicSize{"kernel_index_row_stride"};
|
||||
auto value_stride = SymbolicSize{"value_row_stride"};
|
||||
auto device = SymbolicDevice{};
|
||||
|
||||
CHECK_HOST(topk > 0 && topk <= static_cast<int64_t>(Config::max_topk))
|
||||
<< "topk must be in (0, " << Config::max_topk << "] for this module, got " << topk;
|
||||
CHECK_HOST(!begin.has_value()) << "`begin` is not supported currently";
|
||||
CHECK_HOST(
|
||||
idx_oob_fill_value >= std::numeric_limits<int32_t>::min() &&
|
||||
idx_oob_fill_value <= std::numeric_limits<int32_t>::max())
|
||||
<< "idx_oob_fill_value must fit in int32";
|
||||
CHECK_HOST(input_storage_bytes >= 0) << "input_storage_bytes must be non-negative";
|
||||
|
||||
TensorMatcher({batch, vocab})
|
||||
.with_strides({input_stride, 1})
|
||||
.with_device<kDLCUDA>(device)
|
||||
.with_dtype<ValueT>()
|
||||
.verify(input);
|
||||
TensorMatcher({batch, topk})
|
||||
.with_strides({index_stride, 1})
|
||||
.with_device<kDLCUDA>(device)
|
||||
.with_dtype<OutIdxT>()
|
||||
.verify(output_index);
|
||||
TensorMatcher({batch, topk})
|
||||
.with_strides({kernel_index_stride, 1})
|
||||
.with_device<kDLCUDA>(device)
|
||||
.with_dtype<OutIdxT>()
|
||||
.ensure_alignment(OUTPUT_STRIDE_ALIGNMENT_REQUIREMENT)
|
||||
.verify(kernel_output_index);
|
||||
|
||||
void* output_value_ptr = nullptr;
|
||||
uint64_t stride_output_value = 0;
|
||||
if constexpr (Config::return_value) {
|
||||
CHECK_HOST(output_value.has_value()) << "output_value is required when return_value is enabled";
|
||||
TensorMatcher({batch, topk})
|
||||
.with_strides({value_stride, 1})
|
||||
.with_device<kDLCUDA>(device)
|
||||
.with_dtype<ValueT>()
|
||||
.ensure_alignment(OUTPUT_STRIDE_ALIGNMENT_REQUIREMENT)
|
||||
.verify(output_value.value());
|
||||
output_value_ptr = output_value.value().data_ptr();
|
||||
stride_output_value = static_cast<uint64_t>(value_stride.unwrap());
|
||||
} else {
|
||||
CHECK_HOST(!output_value.has_value()) << "output_value must be absent when return_value is disabled";
|
||||
}
|
||||
|
||||
int* end_ptr = nullptr;
|
||||
if (end.has_value()) {
|
||||
TensorMatcher({batch}).with_device<kDLCUDA>(device).with_dtype<int32_t>().verify(end.value());
|
||||
end_ptr = static_cast<int*>(end.value().data_ptr());
|
||||
}
|
||||
int* index_offset_ptr = nullptr;
|
||||
if (output_index_offset.has_value()) {
|
||||
TensorMatcher({batch}).with_device<kDLCUDA>(device).with_dtype<int32_t>().verify(output_index_offset.value());
|
||||
index_offset_ptr = static_cast<int*>(output_index_offset.value().data_ptr());
|
||||
}
|
||||
|
||||
const auto batch_size = static_cast<uint32_t>(batch.unwrap());
|
||||
const auto vocab_size = static_cast<uint32_t>(vocab.unwrap());
|
||||
CHECK_HOST(vocab_size < MAX_VOCAB_SIZE) << "vocab_size must be < " << MAX_VOCAB_SIZE << ", got " << vocab_size;
|
||||
|
||||
const DLDevice dev = device.unwrap();
|
||||
const auto input_stride_elements = input_stride.unwrap();
|
||||
CHECK_HOST(input_stride_elements >= 0) << "input row stride must be non-negative";
|
||||
CHECK_HOST(static_cast<uint64_t>(input_stride_elements) * sizeof(ValueT) % INPUT_STRIDE_ALIGNMENT_REQUIREMENT == 0)
|
||||
<< "input row stride must be a multiple of " << INPUT_STRIDE_ALIGNMENT_REQUIREMENT << " bytes";
|
||||
CHECK_HOST(static_cast<uint64_t>(index_stride.unwrap()) * sizeof(OutIdxT) % OUTPUT_STRIDE_ALIGNMENT_REQUIREMENT == 0)
|
||||
<< "output_index row stride must be a multiple of " << OUTPUT_STRIDE_ALIGNMENT_REQUIREMENT << " bytes";
|
||||
|
||||
if (batch_size != 0 && vocab_size != 0) {
|
||||
constexpr uint64_t address_alignment = SGL_CUDA_ARCH == 900 ? 16 : 32;
|
||||
CHECK_HOST(reinterpret_cast<uintptr_t>(input.data_ptr()) % address_alignment == 0)
|
||||
<< "input address must be aligned to " << address_alignment << " bytes";
|
||||
const uint64_t row_bytes = static_cast<uint64_t>(vocab_size) * sizeof(ValueT);
|
||||
const uint64_t padded_row_bytes = (row_bytes + 127) / 128 * 128;
|
||||
const uint64_t required_bytes =
|
||||
static_cast<uint64_t>(batch_size - 1) * input_stride_elements * sizeof(ValueT) + padded_row_bytes;
|
||||
CHECK_HOST(required_bytes <= static_cast<uint64_t>(input_storage_bytes))
|
||||
<< "input storage must include the final row padded to 128 bytes";
|
||||
}
|
||||
|
||||
return LaunchPlan{
|
||||
TopkSelectArgs{
|
||||
batch_size,
|
||||
vocab_size,
|
||||
static_cast<uint32_t>(topk),
|
||||
input.data_ptr(),
|
||||
output_value_ptr,
|
||||
kernel_output_index.data_ptr(),
|
||||
nullptr, // `begin` is unimplemented upstream
|
||||
end_ptr,
|
||||
index_offset_ptr,
|
||||
static_cast<uint64_t>(input_stride.unwrap()),
|
||||
stride_output_value,
|
||||
static_cast<uint64_t>(kernel_index_stride.unwrap()),
|
||||
Config::sorted_value,
|
||||
Config::sorted_index,
|
||||
Config::return_value,
|
||||
static_cast<int>(idx_oob_fill_value),
|
||||
static_cast<float>(value_oob_fill_value),
|
||||
abort_when_nan_found,
|
||||
host::runtime::get_max_smem_per_block(dev.device_id),
|
||||
host::LaunchKernel::resolve_device(dev),
|
||||
},
|
||||
dev,
|
||||
};
|
||||
}
|
||||
|
||||
} // namespace details
|
||||
|
||||
/**
|
||||
* \brief Top-k through the one-CTA-per-row kernels.
|
||||
*
|
||||
* \tparam ConfigWave1 Tuning for a batch that fits in a single wave.
|
||||
* \tparam ConfigWaves Tuning for anything larger. A bucket with no wave split
|
||||
* leaves it defaulted; one kernel is then instantiated and the SM-count
|
||||
* query drops out.
|
||||
*/
|
||||
template <typename ConfigWave1, typename ConfigWaves = ConfigWave1>
|
||||
struct TopkNormal {
|
||||
static_assert(ConfigWave1::cluster_size == 1 && ConfigWaves::cluster_size == 1);
|
||||
static_assert(std::is_same_v<typename ConfigWave1::ValueT, typename ConfigWaves::ValueT>);
|
||||
static_assert(std::is_same_v<typename ConfigWave1::OutIdxT, typename ConfigWaves::OutIdxT>);
|
||||
static_assert(ConfigWave1::max_topk == ConfigWaves::max_topk);
|
||||
static_assert(ConfigWave1::sorted_value == ConfigWaves::sorted_value);
|
||||
static_assert(ConfigWave1::sorted_index == ConfigWaves::sorted_index);
|
||||
static_assert(ConfigWave1::return_value == ConfigWaves::return_value);
|
||||
|
||||
static auto topk(
|
||||
tvm::ffi::TensorView input,
|
||||
tvm::ffi::Optional<tvm::ffi::TensorView> output_value,
|
||||
tvm::ffi::TensorView output_index,
|
||||
tvm::ffi::TensorView kernel_output_index,
|
||||
tvm::ffi::Optional<tvm::ffi::TensorView> begin,
|
||||
tvm::ffi::Optional<tvm::ffi::TensorView> end,
|
||||
tvm::ffi::Optional<tvm::ffi::TensorView> output_index_offset,
|
||||
int64_t topk,
|
||||
int64_t input_storage_bytes,
|
||||
int64_t idx_oob_fill_value,
|
||||
double value_oob_fill_value,
|
||||
bool abort_when_nan_found) -> void {
|
||||
const auto plan = details::make_plan<ConfigWave1>(
|
||||
input,
|
||||
output_value,
|
||||
output_index,
|
||||
kernel_output_index,
|
||||
begin,
|
||||
end,
|
||||
output_index_offset,
|
||||
topk,
|
||||
input_storage_bytes,
|
||||
idx_oob_fill_value,
|
||||
value_oob_fill_value,
|
||||
abort_when_nan_found);
|
||||
if (plan.args.batch_size == 0) return;
|
||||
if constexpr (std::is_same_v<ConfigWave1, ConfigWaves>) {
|
||||
details::run_normal<ConfigWave1>(plan.args);
|
||||
} else {
|
||||
const auto num_sm = host::runtime::get_sm_count(plan.device.device_id);
|
||||
if (plan.args.batch_size <= num_sm) {
|
||||
details::run_normal<ConfigWave1>(plan.args);
|
||||
} else {
|
||||
details::run_normal<ConfigWaves>(plan.args);
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* \brief Top-k through the cluster kernel: `Config::cluster_size` CTAs per row.
|
||||
*
|
||||
* Whether a launch belongs here -- small batch, long rows, bf16 -- is decided
|
||||
* in Python, which also picks the cluster size: 8 CTAs on SM90, 16 on
|
||||
* SM100/SM103, where 16 needs the non-portable cluster attribute.
|
||||
*/
|
||||
template <typename Config>
|
||||
struct TopkCluster {
|
||||
static_assert(Config::cluster_size > 1);
|
||||
|
||||
static auto topk(
|
||||
tvm::ffi::TensorView input,
|
||||
tvm::ffi::Optional<tvm::ffi::TensorView> output_value,
|
||||
tvm::ffi::TensorView output_index,
|
||||
tvm::ffi::TensorView kernel_output_index,
|
||||
tvm::ffi::Optional<tvm::ffi::TensorView> begin,
|
||||
tvm::ffi::Optional<tvm::ffi::TensorView> end,
|
||||
tvm::ffi::Optional<tvm::ffi::TensorView> output_index_offset,
|
||||
int64_t topk,
|
||||
int64_t input_storage_bytes,
|
||||
int64_t idx_oob_fill_value,
|
||||
double value_oob_fill_value,
|
||||
bool abort_when_nan_found) -> void {
|
||||
const auto plan = details::make_plan<Config>(
|
||||
input,
|
||||
output_value,
|
||||
output_index,
|
||||
kernel_output_index,
|
||||
begin,
|
||||
end,
|
||||
output_index_offset,
|
||||
topk,
|
||||
input_storage_bytes,
|
||||
idx_oob_fill_value,
|
||||
value_oob_fill_value,
|
||||
abort_when_nan_found);
|
||||
if (plan.args.batch_size == 0) return;
|
||||
topk_select_bf16_cluster::run_topk_select_kernel<Config>(plan.args);
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace deepselect
|
||||
|
||||
} // namespace sglang
|
||||
@@ -0,0 +1,4 @@
|
||||
# Vendored from deepseek-ai/DeepSelect; keep the sources byte-comparable with
|
||||
# upstream so re-syncing stays a plain copy. See README.md.
|
||||
DisableFormat: true
|
||||
SortIncludes: Never
|
||||
@@ -0,0 +1,20 @@
|
||||
# Kerutils
|
||||
|
||||
Kerutils is a library that provides:
|
||||
|
||||
- Wrappers for low-level PTX instructions
|
||||
- Helpers for checking tensor device / shape / stride / dtype, useful in kernel library's dispatcher
|
||||
|
||||
> This copy is a subset of upstream Kerutils: the CuTe UTCMMA / 2SM TMA copy wrappers,
|
||||
> the GeMM helpers and the Kernel Insight (KI) library have been removed because
|
||||
> DeepSelect does not use them.
|
||||
|
||||
## Getting Started
|
||||
|
||||
To use Kerutils, simply clone this repo (or register it as a submodule, if you're working in a repo), and include `<kerutils/kerutils.cuh>`. You should add `include/` into your compiler's `includePath` (for example, by adding `-Ipath/to/kerutils/include`).
|
||||
|
||||
## Organization
|
||||
|
||||
- `include/host` - Host-side helper functions
|
||||
- `include/device` - Device-side PTX wrappers and helper functions, organized by GPU architectures
|
||||
- `supplemental/` - Files in this directory are NOT included by `kerutils/kerutils.cuh` by default, to reduce compilation time. `supplemental/torch_tensors.h` provides various handy guard functions for examining a libTorch tensor.
|
||||
Vendored
+15
@@ -0,0 +1,15 @@
|
||||
#pragma once
|
||||
|
||||
namespace kerutils {}
|
||||
|
||||
#define KU_PRINTLN(fmt, ...) { cute::print(fmt, ##__VA_ARGS__); print("\n"); }
|
||||
|
||||
namespace ku = kerutils;
|
||||
|
||||
#ifdef __CUDACC__
|
||||
#define KERUTILS_IS_BUILD_ON_CUDA
|
||||
#endif
|
||||
|
||||
#ifndef KERUTILS_IS_BUILD_ON_CUDA
|
||||
#error "KERUTILS_IS_BUILD_ON_CUDA must be defined. It is defined automatically when compiling with NVCC; pass `-DKERUTILS_IS_BUILD_ON_CUDA` when compiling with a host compiler."
|
||||
#endif
|
||||
+75
@@ -0,0 +1,75 @@
|
||||
/*
|
||||
Common data types and macros that are used across the kerutils library.
|
||||
*/
|
||||
#pragma once
|
||||
|
||||
#include <cuda_bf16.h>
|
||||
#include <cuda_fp8.h>
|
||||
|
||||
#include <cutlass/bfloat16.h>
|
||||
#include <cutlass/arch/barrier.h>
|
||||
#include <cute/config.hpp> // For CUTE_DEVICE
|
||||
|
||||
namespace kerutils {
|
||||
|
||||
// Cache hints
|
||||
enum class CacheHint {
|
||||
EVICT_FIRST,
|
||||
EVICT_NORMAL,
|
||||
EVICT_LAST,
|
||||
EVICT_UNCHANGED,
|
||||
NO_ALLOCATE
|
||||
};
|
||||
|
||||
// Prefetch size
|
||||
enum class PrefetchSize {
|
||||
B64,
|
||||
B128,
|
||||
B256
|
||||
};
|
||||
|
||||
using nvbf16 = __nv_bfloat16;
|
||||
using nvbf16x2 = __nv_bfloat162;
|
||||
using nve4m3 = __nv_fp8_e4m3;
|
||||
using nve4m3x2 = __nv_fp8x2_e4m3;
|
||||
using nve4m3x4 = __nv_fp8x4_e4m3;
|
||||
|
||||
using bf16 = cutlass::bfloat16_t;
|
||||
using transac_bar_t = cutlass::arch::ClusterTransactionBarrier;
|
||||
|
||||
}
|
||||
|
||||
#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 800))
|
||||
#define KERUTILS_ENABLE_SM80
|
||||
#elif (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ < 800))
|
||||
static_assert(false, "kerutils doesn't support SM architectures below SM80");
|
||||
#endif
|
||||
|
||||
#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900))
|
||||
#define KERUTILS_ENABLE_SM90
|
||||
#endif
|
||||
|
||||
#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900 && __CUDA_ARCH__ < 1000))
|
||||
#define KERUTILS_ENABLE_SM90A
|
||||
#endif
|
||||
|
||||
#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000))
|
||||
#define KERUTILS_ENABLE_SM100
|
||||
#endif
|
||||
|
||||
#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000 && __CUDA_ARCH__ < 1200))
|
||||
#define KERUTILS_ENABLE_SM100A
|
||||
#endif
|
||||
|
||||
#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1030 && __CUDA_ARCH__ < 1200))
|
||||
#define KERUTILS_ENABLE_SM103A
|
||||
#endif
|
||||
|
||||
#if (defined(__CLION_IDE__) || defined(__VSCODE_IDE__))
|
||||
#define KERUTILS_ENABLE_SM80
|
||||
#define KERUTILS_ENABLE_SM90
|
||||
#define KERUTILS_ENABLE_SM90A
|
||||
#define KERUTILS_ENABLE_SM100
|
||||
#define KERUTILS_ENABLE_SM100A
|
||||
#define KERUTILS_ENABLE_SM103A
|
||||
#endif
|
||||
+649
@@ -0,0 +1,649 @@
|
||||
#pragma once
|
||||
|
||||
#include <cute/atom/copy_traits_sm100.hpp>
|
||||
|
||||
#include "kerutils/device/cuda/common.h"
|
||||
|
||||
namespace kerutils {
|
||||
|
||||
// tma gather4 (https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-cp-async-bulk-tensor)
|
||||
// Please pay attention that the coordinates of TMA gather4 are int32, which may lead to overflow under some scenarios
|
||||
CUTE_DEVICE
|
||||
void tma_gather4(const void* desc_ptr, transac_bar_t &mbar_ptr, void* smem_ptr, int col_idx, int4 row_idxs, int64_t cache_hint) {
|
||||
uint32_t smem_addr = cute::cast_smem_ptr_to_uint(smem_ptr);
|
||||
uint32_t mbar_addr = cute::cast_smem_ptr_to_uint(&mbar_ptr);
|
||||
asm volatile(
|
||||
"cp.async.bulk.tensor.2d.shared::cta.global.tile::gather4.mbarrier::complete_tx::bytes.cta_group::1.L2::cache_hint [%0], [%1, {%2, %3, %4, %5, %6}], [%7], %8;\n"
|
||||
:
|
||||
: "r"(smem_addr), "l"(desc_ptr), "r"(col_idx),
|
||||
"r"(row_idxs.x), "r"(row_idxs.y), "r"(row_idxs.z), "r"(row_idxs.w),
|
||||
"r"(mbar_addr), "l"(cache_hint)
|
||||
: "memory"
|
||||
);
|
||||
}
|
||||
|
||||
// tma gather4 prefetch (https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-cp-async-bulk-prefetch-tensor)
|
||||
// Please pay attention that the coordinates of TMA gather4 are int32, which may lead to overflow under some scenarios
|
||||
CUTE_DEVICE
|
||||
void tma_gather4_prefetch(const void* desc_ptr, int col_idx, int4 row_idxs, int64_t cache_hint) {
|
||||
asm volatile(
|
||||
"cp.async.bulk.prefetch.tensor.2d.L2.global.tile::gather4.L2::cache_hint [%0, {%1, %2, %3, %4, %5}], %6;\n"
|
||||
:
|
||||
: "l"(desc_ptr), "r"(col_idx),
|
||||
"r"(row_idxs.x), "r"(row_idxs.y), "r"(row_idxs.z), "r"(row_idxs.w),
|
||||
"l"(cache_hint)
|
||||
);
|
||||
}
|
||||
|
||||
// tma gather4 with cta_group::2, allowing for synchronization across CTAs within a pair of CTAs (https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-cp-async-bulk-tensor)
|
||||
template<bool USE_CTA0_MBAR = false>
|
||||
CUTE_DEVICE void tma_gather4_cta_group_2(const void* desc_ptr, transac_bar_t &mbar_ptr, void* smem_ptr, int col_idx, int4 row_idxs, int64_t cache_hint) {
|
||||
uint32_t smem_addr = cute::cast_smem_ptr_to_uint(smem_ptr);
|
||||
uint32_t mbar_addr = cute::cast_smem_ptr_to_uint(&mbar_ptr);
|
||||
if constexpr (USE_CTA0_MBAR) {
|
||||
mbar_addr &= cute::Sm100MmaPeerBitMask;
|
||||
}
|
||||
asm volatile(
|
||||
"cp.async.bulk.tensor.2d.shared::cta.global.tile::gather4.mbarrier::complete_tx::bytes.cta_group::2.L2::cache_hint [%0], [%1, {%2, %3, %4, %5, %6}], [%7], %8;\n"
|
||||
:
|
||||
: "r"(smem_addr), "l"(desc_ptr), "r"(col_idx),
|
||||
"r"(row_idxs.x), "r"(row_idxs.y), "r"(row_idxs.z), "r"(row_idxs.w),
|
||||
"r"(mbar_addr), "l"(cache_hint)
|
||||
: "memory"
|
||||
);
|
||||
}
|
||||
|
||||
// Vectorized addition for float32 (https://docs.nvidia.com/cuda/parallel-thread-execution/#floating-point-instructions-add)
|
||||
CUTE_DEVICE
|
||||
float2 float2_add(const float2 &a, const float2 &b) {
|
||||
float2 c;
|
||||
asm volatile(
|
||||
"add.f32x2 %0, %1, %2;\n"
|
||||
: "=l"(reinterpret_cast<uint64_t&>(c))
|
||||
: "l"(reinterpret_cast<uint64_t const&>(a)),
|
||||
"l"(reinterpret_cast<uint64_t const&>(b))
|
||||
);
|
||||
return c;
|
||||
}
|
||||
|
||||
// Vectorized multiplication for float32 (https://docs.nvidia.com/cuda/parallel-thread-execution/#floating-point-instructions-mul)
|
||||
CUTE_DEVICE
|
||||
float2 float2_mul(const float2 &a, const float2 &b) {
|
||||
float2 c;
|
||||
asm volatile(
|
||||
"mul.f32x2 %0, %1, %2;\n"
|
||||
: "=l"(reinterpret_cast<uint64_t&>(c))
|
||||
: "l"(reinterpret_cast<uint64_t const&>(a)),
|
||||
"l"(reinterpret_cast<uint64_t const&>(b)));
|
||||
return c;
|
||||
}
|
||||
|
||||
// Vectorized fused addition-multiplication for float32 (https://docs.nvidia.com/cuda/parallel-thread-execution/#floating-point-instructions-fma)
|
||||
CUTE_DEVICE
|
||||
float2 float2_fma(const float2 &a, const float2 &b, const float2 &c) {
|
||||
// return a*b+c
|
||||
float2 d;
|
||||
asm volatile(
|
||||
"fma.rn.f32x2 %0, %1, %2, %3;\n"
|
||||
: "=l"(reinterpret_cast<uint64_t&>(d))
|
||||
: "l"(reinterpret_cast<uint64_t const&>(a)),
|
||||
"l"(reinterpret_cast<uint64_t const&>(b)),
|
||||
"l"(reinterpret_cast<uint64_t const&>(c)));
|
||||
return d;
|
||||
}
|
||||
|
||||
// Vectorized negation for foat32
|
||||
CUTE_DEVICE
|
||||
float2 float2_neg(const float2 &a) {
|
||||
float2 t = {-1.0f, -1.0f};
|
||||
return float2_mul(a, t);
|
||||
}
|
||||
|
||||
// st.bulk (https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-st-bulk)
|
||||
CUTE_DEVICE
|
||||
void st_bulk(void* dst_ptr, int64_t size) {
|
||||
uint32_t dst_addr = cute::cast_smem_ptr_to_uint(dst_ptr);
|
||||
asm volatile (
|
||||
"st.bulk.weak.shared::cta [%0], %1, 0;\n"
|
||||
:
|
||||
: "r"(dst_addr), "l"(size)
|
||||
: "memory"
|
||||
);
|
||||
}
|
||||
|
||||
struct CUTE_ALIGNAS(16) CLCResponseObj {
|
||||
// An opaque 16B value
|
||||
char opaque[16];
|
||||
};
|
||||
|
||||
struct CLCResult {
|
||||
int is_valid;
|
||||
int x, y, z;
|
||||
};
|
||||
|
||||
// Issue a CLC try_cancel query (https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-try-cancel)
|
||||
CUTE_DEVICE
|
||||
void issue_clc_query(transac_bar_t &bar, CLCResponseObj &response_obj) {
|
||||
uint32_t response_addr = cute::cast_smem_ptr_to_uint(response_obj.opaque);
|
||||
uint32_t mbarrier_addr = cute::cast_smem_ptr_to_uint(&bar);
|
||||
asm volatile(
|
||||
"clusterlaunchcontrol.try_cancel.async.shared::cta.mbarrier::complete_tx::bytes.b128 [%0], [%1];\n"
|
||||
:
|
||||
: "r"(response_addr), "r"(mbarrier_addr)
|
||||
);
|
||||
}
|
||||
|
||||
// Issue a CLC try_cancel query with .multicast::cluster::all (https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-try-cancel)
|
||||
CUTE_DEVICE
|
||||
void issue_clc_query_multicast_cluster_all(transac_bar_t &bar, CLCResponseObj &response_obj) {
|
||||
uint32_t response_addr = cute::cast_smem_ptr_to_uint(response_obj.opaque);
|
||||
uint32_t mbarrier_addr = cute::cast_smem_ptr_to_uint(&bar);
|
||||
asm volatile(
|
||||
"clusterlaunchcontrol.try_cancel.async.shared::cta.mbarrier::complete_tx::bytes.multicast::cluster::all.b128 [%0], [%1];\n"
|
||||
:
|
||||
: "r"(response_addr), "r"(mbarrier_addr)
|
||||
);
|
||||
}
|
||||
|
||||
// Get the result of a CLC query (https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-query-cancel)
|
||||
// In this function, we separate get_first_ctaid::x/y/z and hope PTXAS's dead code elimination can remove unnecessary instructions
|
||||
template<bool USE_LD_ACQUIRE>
|
||||
CUTE_DEVICE
|
||||
CLCResult get_clc_query_response(CLCResponseObj &response_obj) {
|
||||
uint32_t response_addr = cute::cast_smem_ptr_to_uint(&response_obj);
|
||||
CLCResult result;
|
||||
#define EMIT_ASM(LD_MODIFIER) \
|
||||
asm volatile( \
|
||||
"{\n" \
|
||||
".reg .pred p1;\n\t" \
|
||||
".reg .b128 clc_result;\n\t" \
|
||||
"ld" LD_MODIFIER ".shared.b128 clc_result, [%4];\n\t" \
|
||||
"clusterlaunchcontrol.query_cancel.is_canceled.pred.b128 p1, clc_result;\n\t" \
|
||||
"selp.u32 %3, 1, 0, p1;\n\t" \
|
||||
"@p1 clusterlaunchcontrol.query_cancel.get_first_ctaid::x.b32.b128 %0, clc_result;\n\t" \
|
||||
"@p1 clusterlaunchcontrol.query_cancel.get_first_ctaid::y.b32.b128 %1, clc_result;\n\t" \
|
||||
"@p1 clusterlaunchcontrol.query_cancel.get_first_ctaid::z.b32.b128 %2, clc_result;\n\t" \
|
||||
"}\n" \
|
||||
: "=r"(result.x), "=r"(result.y), "=r"(result.z), "=r"(result.is_valid) \
|
||||
: "r"(response_addr) \
|
||||
: "memory" \
|
||||
);
|
||||
if constexpr (USE_LD_ACQUIRE) {
|
||||
EMIT_ASM(".acquire.cta");
|
||||
} else {
|
||||
EMIT_ASM("");
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
// LDG.256 or LDG.256 with non-coherent cache (https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-ld)
|
||||
// We use macro instead of function here, since we need a multi-level recursive dispatch based on template parameters if using function
|
||||
// NC_STR should be either "" or ".nc"
|
||||
// L1_CACHE_HINT_STR should be either "evict_first", "evict_normal", "evict_last", "evict_unchanged", or "no_allocate"
|
||||
// L2_CACHE_HINT_STR should be either "evict_first", "evict_normal", or "evict_last"
|
||||
// L2_PREFETCH_SIZE_STR should be either "64B", "128B", or "256B"
|
||||
#define KU_LDG_256(global_addr, result, NC_STR, L1_CACHE_HINT_STR, L2_CACHE_HINT_STR, L2_PREFETCH_SIZE_STR) \
|
||||
{ \
|
||||
static_assert(std::is_pointer_v<decltype(global_addr)> || std::is_array_v<decltype(global_addr)>, "`global_addr` must be a pointer"); \
|
||||
static_assert(std::is_pointer_v<decltype(result)> || std::is_array_v<decltype(result)>, "`result` must be a pointer"); \
|
||||
uint64_t* result_as_uint64_ptr = (uint64_t*)(result); \
|
||||
asm volatile( \
|
||||
"ld.global" NC_STR ".L1::" L1_CACHE_HINT_STR ".L2::" L2_CACHE_HINT_STR ".L2::" L2_PREFETCH_SIZE_STR ".v4.u64 {%0, %1, %2, %3}, [%4];\n" \
|
||||
: "=l"(result_as_uint64_ptr[0]), "=l"(result_as_uint64_ptr[1]), \
|
||||
"=l"(result_as_uint64_ptr[2]), "=l"(result_as_uint64_ptr[3]) \
|
||||
: "l"(global_addr) \
|
||||
); \
|
||||
}
|
||||
|
||||
// STG.256 (https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-st)
|
||||
// L1_CACHE_HINT_STR should be either "evict_first", "evict_normal", "evict_last", "evict_unchanged", or "no_allocate"
|
||||
// L2_CACHE_HINT_STR should be either "evict_first", "evict_normal", or "evict_last"
|
||||
#define KU_STG_256(global_addr, src, L1_CACHE_HINT_STR, L2_CACHE_HINT_STR) \
|
||||
{ \
|
||||
static_assert(std::is_pointer_v<decltype(global_addr)> || std::is_array_v<decltype(global_addr)>, "`global_addr` must be a pointer"); \
|
||||
static_assert(std::is_pointer_v<decltype(src)> || std::is_array_v<decltype(src)>, "`src` must be a pointer"); \
|
||||
uint64_t const* src_as_uint64_ptr = (uint64_t const*)(src); \
|
||||
asm volatile( \
|
||||
"st.global.L1::" L1_CACHE_HINT_STR ".L2::" L2_CACHE_HINT_STR ".v4.u64 [%0], {%1, %2, %3, %4};\n" \
|
||||
: \
|
||||
: "l"(global_addr), "l"(src_as_uint64_ptr[0]), "l"(src_as_uint64_ptr[1]), \
|
||||
"l"(src_as_uint64_ptr[2]), "l"(src_as_uint64_ptr[3]) \
|
||||
); \
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
namespace kerutils {
|
||||
|
||||
// tcgen05.commit.cta_group::1 (https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen-async-sync-operations-commit)
|
||||
CUTE_DEVICE
|
||||
void umma_arrive_noelect(transac_bar_t &bar) {
|
||||
uint32_t bar_intptr = cute::cast_smem_ptr_to_uint(&bar);
|
||||
asm volatile(
|
||||
"tcgen05.commit.cta_group::1.mbarrier::arrive::one.shared::cluster.b64 [%0];\n"
|
||||
:
|
||||
:"r"(bar_intptr)
|
||||
);
|
||||
}
|
||||
|
||||
// tcgen05.commit.cta_group::1, with multicast (https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen-async-sync-operations-commit)
|
||||
CUTE_DEVICE
|
||||
void umma_arrive_multicast_noelect(transac_bar_t &bar, uint16_t cta_mask) {
|
||||
uint32_t bar_intptr = cute::cast_smem_ptr_to_uint(&bar);
|
||||
asm volatile(
|
||||
"tcgen05.commit.cta_group::1.mbarrier::arrive::one.shared::cluster.multicast::cluster.b64 [%0], %1;\n"
|
||||
:
|
||||
:"r"(bar_intptr), "h"(cta_mask)
|
||||
);
|
||||
}
|
||||
|
||||
// tcgen05.commit.cta_group::2 (https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen-async-sync-operations-commit)
|
||||
CUTE_DEVICE
|
||||
void umma_arrive_2x1SM_noelect(transac_bar_t &bar) {
|
||||
uint32_t bar_intptr = cute::cast_smem_ptr_to_uint(&bar);
|
||||
asm volatile(
|
||||
"tcgen05.commit.cta_group::2.mbarrier::arrive::one.shared::cluster.b64 [%0];\n"
|
||||
:
|
||||
:"r"(bar_intptr)
|
||||
);
|
||||
}
|
||||
|
||||
// tcgen05.commit.cta_group::2, with multicast (https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen-async-sync-operations-commit)
|
||||
CUTE_DEVICE
|
||||
void umma_arrive_multicast_2x1SM_noelect(transac_bar_t &bar, uint16_t cta_mask) {
|
||||
uint32_t bar_intptr = cute::cast_smem_ptr_to_uint(&bar);
|
||||
asm volatile(
|
||||
"tcgen05.commit.cta_group::2.mbarrier::arrive::one.shared::cluster.multicast::cluster.b64 [%0], %1;\n"
|
||||
:
|
||||
:"r"(bar_intptr), "h"(cta_mask)
|
||||
);
|
||||
}
|
||||
|
||||
// tcgen05.fence::before_thread_sync (https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-special-sync-operations-fence)
|
||||
__device__ __forceinline__ void tcgen05_before_thread_sync() {
|
||||
asm volatile("tcgen05.fence::before_thread_sync;");
|
||||
}
|
||||
|
||||
// tcgen05.fence::after_thread_sync (https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-special-sync-operations-fence)
|
||||
__device__ __forceinline__ void tcgen05_after_thread_sync() {
|
||||
asm volatile("tcgen05.fence::after_thread_sync;");
|
||||
}
|
||||
|
||||
|
||||
// Load from tensor memory, 32 data path lanes, 32-bit pattern, repeated N times. (https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-instructions-tcgen05-ld)
|
||||
template <int kNumElements>
|
||||
__device__ __forceinline__
|
||||
void tmem_ld_32dp32bNx(uint32_t tmem_start, void* data_) {
|
||||
uint32_t* data = (uint32_t*)data_;
|
||||
static_assert(kNumElements == 1 || kNumElements == 2 || kNumElements == 4 || kNumElements == 8 || kNumElements == 16 || kNumElements == 32 || kNumElements == 64 || kNumElements == 128, "Invalid kNumElements");
|
||||
// NOTE The following code crashes VSCode intellisense engine, so we disable it
|
||||
#ifndef __VSCODE_IDE__
|
||||
[&]<size_t... Is>(cute::index_sequence<Is...>) {
|
||||
if constexpr (kNumElements == 1) {
|
||||
cute::SM100_TMEM_LOAD_32dp32b1x::copy(tmem_start, data[Is]...);
|
||||
} else if constexpr (kNumElements == 2) {
|
||||
cute::SM100_TMEM_LOAD_32dp32b2x::copy(tmem_start, data[Is]...);
|
||||
} else if constexpr (kNumElements == 4) {
|
||||
cute::SM100_TMEM_LOAD_32dp32b4x::copy(tmem_start, data[Is]...);
|
||||
} else if constexpr (kNumElements == 8) {
|
||||
cute::SM100_TMEM_LOAD_32dp32b8x::copy(tmem_start, data[Is]...);
|
||||
} else if constexpr (kNumElements == 16) {
|
||||
cute::SM100_TMEM_LOAD_32dp32b16x::copy(tmem_start, data[Is]...);
|
||||
} else if constexpr (kNumElements == 32) {
|
||||
cute::SM100_TMEM_LOAD_32dp32b32x::copy(tmem_start, data[Is]...);
|
||||
} else if constexpr (kNumElements == 64) {
|
||||
cute::SM100_TMEM_LOAD_32dp32b64x::copy(tmem_start, data[Is]...);
|
||||
} else if constexpr (kNumElements == 128) {
|
||||
cute::SM100_TMEM_LOAD_32dp32b128x::copy(tmem_start, data[Is]...);
|
||||
}
|
||||
}(cute::make_index_sequence<kNumElements>{});
|
||||
#endif
|
||||
}
|
||||
|
||||
// Load from tensor memory with column-wise reduction, 32 data path lanes, 32-bit pattern, repeated N times.
|
||||
// (https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-instructions-tcgen05-ld)
|
||||
// USE_MAX: true = .max reduction, false = .min reduction
|
||||
// USE_ABS: true = .abs qualifier (use absolute values for reduction)
|
||||
// USE_NAN: true = .NaN qualifier (propagate NaN in reduction)
|
||||
template <int kNumElements, bool USE_MAX, bool USE_ABS, bool USE_NAN>
|
||||
__device__ __forceinline__
|
||||
void tmem_ld_red_32dp32bNx(uint32_t tmem_start, void* data_, float& redval) {
|
||||
uint32_t* data = (uint32_t*)data_;
|
||||
static_assert(kNumElements == 2 || kNumElements == 4 || kNumElements == 8 ||
|
||||
kNumElements == 16 || kNumElements == 32 || kNumElements == 64 ||
|
||||
kNumElements == 128,
|
||||
"Invalid kNumElements for tcgen05.ld.red (must be power of 2, at least 2)");
|
||||
#ifndef __VSCODE_IDE__
|
||||
static constexpr char s_max[] = ".max";
|
||||
static constexpr char s_min[] = ".min";
|
||||
static constexpr char s_abs[] = ".abs";
|
||||
static constexpr char s_nan[] = ".NaN";
|
||||
static constexpr char s_empty[] = "";
|
||||
// Operand numbering for each branch:
|
||||
// outputs: %0..%{N-1} = data[0..N-1] ("=r"), %N = redval ("=f")
|
||||
// inputs: %{N+1} = tmem_start ("r"), %{N+2} = redop ("C"), %{N+3} = abs ("C"), %{N+4} = nan ("C")
|
||||
if constexpr (kNumElements == 2) {
|
||||
// outputs: %0..%1 = data, %2 = redval; inputs: %3 = taddr, %4 = redop, %5 = abs, %6 = nan
|
||||
asm volatile(
|
||||
"tcgen05.ld.red.sync.aligned.32x32b.x2%4%5%6.f32"
|
||||
" {%0, %1}, %2, [%3];\n"
|
||||
: "=r"(data[0]), "=r"(data[1]),
|
||||
"=f"(redval)
|
||||
: "r"(tmem_start),
|
||||
"C"(USE_MAX ? s_max : s_min),
|
||||
"C"(USE_ABS ? s_abs : s_empty),
|
||||
"C"(USE_NAN ? s_nan : s_empty)
|
||||
);
|
||||
} else if constexpr (kNumElements == 4) {
|
||||
// outputs: %0..%3 = data, %4 = redval; inputs: %5 = taddr, %6..%8 = C
|
||||
asm volatile(
|
||||
"tcgen05.ld.red.sync.aligned.32x32b.x4%6%7%8.f32"
|
||||
" {%0, %1, %2, %3}, %4, [%5];\n"
|
||||
: "=r"(data[0]), "=r"(data[1]), "=r"(data[2]), "=r"(data[3]),
|
||||
"=f"(redval)
|
||||
: "r"(tmem_start),
|
||||
"C"(USE_MAX ? s_max : s_min),
|
||||
"C"(USE_ABS ? s_abs : s_empty),
|
||||
"C"(USE_NAN ? s_nan : s_empty)
|
||||
);
|
||||
} else if constexpr (kNumElements == 8) {
|
||||
// outputs: %0..%7 = data, %8 = redval; inputs: %9 = taddr, %10..%12 = C
|
||||
asm volatile(
|
||||
"tcgen05.ld.red.sync.aligned.32x32b.x8%10%11%12.f32"
|
||||
" {%0, %1, %2, %3,"
|
||||
" %4, %5, %6, %7}, %8, [%9];\n"
|
||||
: "=r"(data[0]), "=r"(data[1]), "=r"(data[2]), "=r"(data[3]),
|
||||
"=r"(data[4]), "=r"(data[5]), "=r"(data[6]), "=r"(data[7]),
|
||||
"=f"(redval)
|
||||
: "r"(tmem_start),
|
||||
"C"(USE_MAX ? s_max : s_min),
|
||||
"C"(USE_ABS ? s_abs : s_empty),
|
||||
"C"(USE_NAN ? s_nan : s_empty)
|
||||
);
|
||||
} else if constexpr (kNumElements == 16) {
|
||||
// outputs: %0..%15 = data, %16 = redval; inputs: %17 = taddr, %18..%20 = C
|
||||
asm volatile(
|
||||
"tcgen05.ld.red.sync.aligned.32x32b.x16%18%19%20.f32"
|
||||
" {%0, %1, %2, %3,"
|
||||
" %4, %5, %6, %7,"
|
||||
" %8, %9, %10, %11,"
|
||||
" %12, %13, %14, %15}, %16, [%17];\n"
|
||||
: "=r"(data[0]), "=r"(data[1]), "=r"(data[2]), "=r"(data[3]),
|
||||
"=r"(data[4]), "=r"(data[5]), "=r"(data[6]), "=r"(data[7]),
|
||||
"=r"(data[8]), "=r"(data[9]), "=r"(data[10]), "=r"(data[11]),
|
||||
"=r"(data[12]), "=r"(data[13]), "=r"(data[14]), "=r"(data[15]),
|
||||
"=f"(redval)
|
||||
: "r"(tmem_start),
|
||||
"C"(USE_MAX ? s_max : s_min),
|
||||
"C"(USE_ABS ? s_abs : s_empty),
|
||||
"C"(USE_NAN ? s_nan : s_empty)
|
||||
);
|
||||
} else if constexpr (kNumElements == 32) {
|
||||
// outputs: %0..%31 = data, %32 = redval; inputs: %33 = taddr, %34..%36 = C
|
||||
asm volatile(
|
||||
"tcgen05.ld.red.sync.aligned.32x32b.x32%34%35%36.f32"
|
||||
" {%0, %1, %2, %3,"
|
||||
" %4, %5, %6, %7,"
|
||||
" %8, %9, %10, %11,"
|
||||
" %12, %13, %14, %15,"
|
||||
" %16, %17, %18, %19,"
|
||||
" %20, %21, %22, %23,"
|
||||
" %24, %25, %26, %27,"
|
||||
" %28, %29, %30, %31}, %32, [%33];\n"
|
||||
: "=r"(data[0]), "=r"(data[1]), "=r"(data[2]), "=r"(data[3]),
|
||||
"=r"(data[4]), "=r"(data[5]), "=r"(data[6]), "=r"(data[7]),
|
||||
"=r"(data[8]), "=r"(data[9]), "=r"(data[10]), "=r"(data[11]),
|
||||
"=r"(data[12]), "=r"(data[13]), "=r"(data[14]), "=r"(data[15]),
|
||||
"=r"(data[16]), "=r"(data[17]), "=r"(data[18]), "=r"(data[19]),
|
||||
"=r"(data[20]), "=r"(data[21]), "=r"(data[22]), "=r"(data[23]),
|
||||
"=r"(data[24]), "=r"(data[25]), "=r"(data[26]), "=r"(data[27]),
|
||||
"=r"(data[28]), "=r"(data[29]), "=r"(data[30]), "=r"(data[31]),
|
||||
"=f"(redval)
|
||||
: "r"(tmem_start),
|
||||
"C"(USE_MAX ? s_max : s_min),
|
||||
"C"(USE_ABS ? s_abs : s_empty),
|
||||
"C"(USE_NAN ? s_nan : s_empty)
|
||||
);
|
||||
} else if constexpr (kNumElements == 64) {
|
||||
// outputs: %0..%63 = data, %64 = redval; inputs: %65 = taddr, %66..%68 = C
|
||||
asm volatile(
|
||||
"tcgen05.ld.red.sync.aligned.32x32b.x64%66%67%68.f32"
|
||||
" {%0, %1, %2, %3,"
|
||||
" %4, %5, %6, %7,"
|
||||
" %8, %9, %10, %11,"
|
||||
" %12, %13, %14, %15,"
|
||||
" %16, %17, %18, %19,"
|
||||
" %20, %21, %22, %23,"
|
||||
" %24, %25, %26, %27,"
|
||||
" %28, %29, %30, %31,"
|
||||
" %32, %33, %34, %35,"
|
||||
" %36, %37, %38, %39,"
|
||||
" %40, %41, %42, %43,"
|
||||
" %44, %45, %46, %47,"
|
||||
" %48, %49, %50, %51,"
|
||||
" %52, %53, %54, %55,"
|
||||
" %56, %57, %58, %59,"
|
||||
" %60, %61, %62, %63}, %64, [%65];\n"
|
||||
: "=r"(data[0]), "=r"(data[1]), "=r"(data[2]), "=r"(data[3]),
|
||||
"=r"(data[4]), "=r"(data[5]), "=r"(data[6]), "=r"(data[7]),
|
||||
"=r"(data[8]), "=r"(data[9]), "=r"(data[10]), "=r"(data[11]),
|
||||
"=r"(data[12]), "=r"(data[13]), "=r"(data[14]), "=r"(data[15]),
|
||||
"=r"(data[16]), "=r"(data[17]), "=r"(data[18]), "=r"(data[19]),
|
||||
"=r"(data[20]), "=r"(data[21]), "=r"(data[22]), "=r"(data[23]),
|
||||
"=r"(data[24]), "=r"(data[25]), "=r"(data[26]), "=r"(data[27]),
|
||||
"=r"(data[28]), "=r"(data[29]), "=r"(data[30]), "=r"(data[31]),
|
||||
"=r"(data[32]), "=r"(data[33]), "=r"(data[34]), "=r"(data[35]),
|
||||
"=r"(data[36]), "=r"(data[37]), "=r"(data[38]), "=r"(data[39]),
|
||||
"=r"(data[40]), "=r"(data[41]), "=r"(data[42]), "=r"(data[43]),
|
||||
"=r"(data[44]), "=r"(data[45]), "=r"(data[46]), "=r"(data[47]),
|
||||
"=r"(data[48]), "=r"(data[49]), "=r"(data[50]), "=r"(data[51]),
|
||||
"=r"(data[52]), "=r"(data[53]), "=r"(data[54]), "=r"(data[55]),
|
||||
"=r"(data[56]), "=r"(data[57]), "=r"(data[58]), "=r"(data[59]),
|
||||
"=r"(data[60]), "=r"(data[61]), "=r"(data[62]), "=r"(data[63]),
|
||||
"=f"(redval)
|
||||
: "r"(tmem_start),
|
||||
"C"(USE_MAX ? s_max : s_min),
|
||||
"C"(USE_ABS ? s_abs : s_empty),
|
||||
"C"(USE_NAN ? s_nan : s_empty)
|
||||
);
|
||||
} else if constexpr (kNumElements == 128) {
|
||||
// outputs: %0..%127 = data, %128 = redval; inputs: %129 = taddr, %130..%132 = C
|
||||
asm volatile(
|
||||
"tcgen05.ld.red.sync.aligned.32x32b.x128%130%131%132.f32"
|
||||
" {%0, %1, %2, %3,"
|
||||
" %4, %5, %6, %7,"
|
||||
" %8, %9, %10, %11,"
|
||||
" %12, %13, %14, %15,"
|
||||
" %16, %17, %18, %19,"
|
||||
" %20, %21, %22, %23,"
|
||||
" %24, %25, %26, %27,"
|
||||
" %28, %29, %30, %31,"
|
||||
" %32, %33, %34, %35,"
|
||||
" %36, %37, %38, %39,"
|
||||
" %40, %41, %42, %43,"
|
||||
" %44, %45, %46, %47,"
|
||||
" %48, %49, %50, %51,"
|
||||
" %52, %53, %54, %55,"
|
||||
" %56, %57, %58, %59,"
|
||||
" %60, %61, %62, %63,"
|
||||
" %64, %65, %66, %67,"
|
||||
" %68, %69, %70, %71,"
|
||||
" %72, %73, %74, %75,"
|
||||
" %76, %77, %78, %79,"
|
||||
" %80, %81, %82, %83,"
|
||||
" %84, %85, %86, %87,"
|
||||
" %88, %89, %90, %91,"
|
||||
" %92, %93, %94, %95,"
|
||||
" %96, %97, %98, %99,"
|
||||
" %100, %101, %102, %103,"
|
||||
" %104, %105, %106, %107,"
|
||||
" %108, %109, %110, %111,"
|
||||
" %112, %113, %114, %115,"
|
||||
" %116, %117, %118, %119,"
|
||||
" %120, %121, %122, %123,"
|
||||
" %124, %125, %126, %127}, %128, [%129];\n"
|
||||
: "=r"(data[0]), "=r"(data[1]), "=r"(data[2]), "=r"(data[3]),
|
||||
"=r"(data[4]), "=r"(data[5]), "=r"(data[6]), "=r"(data[7]),
|
||||
"=r"(data[8]), "=r"(data[9]), "=r"(data[10]), "=r"(data[11]),
|
||||
"=r"(data[12]), "=r"(data[13]), "=r"(data[14]), "=r"(data[15]),
|
||||
"=r"(data[16]), "=r"(data[17]), "=r"(data[18]), "=r"(data[19]),
|
||||
"=r"(data[20]), "=r"(data[21]), "=r"(data[22]), "=r"(data[23]),
|
||||
"=r"(data[24]), "=r"(data[25]), "=r"(data[26]), "=r"(data[27]),
|
||||
"=r"(data[28]), "=r"(data[29]), "=r"(data[30]), "=r"(data[31]),
|
||||
"=r"(data[32]), "=r"(data[33]), "=r"(data[34]), "=r"(data[35]),
|
||||
"=r"(data[36]), "=r"(data[37]), "=r"(data[38]), "=r"(data[39]),
|
||||
"=r"(data[40]), "=r"(data[41]), "=r"(data[42]), "=r"(data[43]),
|
||||
"=r"(data[44]), "=r"(data[45]), "=r"(data[46]), "=r"(data[47]),
|
||||
"=r"(data[48]), "=r"(data[49]), "=r"(data[50]), "=r"(data[51]),
|
||||
"=r"(data[52]), "=r"(data[53]), "=r"(data[54]), "=r"(data[55]),
|
||||
"=r"(data[56]), "=r"(data[57]), "=r"(data[58]), "=r"(data[59]),
|
||||
"=r"(data[60]), "=r"(data[61]), "=r"(data[62]), "=r"(data[63]),
|
||||
"=r"(data[64]), "=r"(data[65]), "=r"(data[66]), "=r"(data[67]),
|
||||
"=r"(data[68]), "=r"(data[69]), "=r"(data[70]), "=r"(data[71]),
|
||||
"=r"(data[72]), "=r"(data[73]), "=r"(data[74]), "=r"(data[75]),
|
||||
"=r"(data[76]), "=r"(data[77]), "=r"(data[78]), "=r"(data[79]),
|
||||
"=r"(data[80]), "=r"(data[81]), "=r"(data[82]), "=r"(data[83]),
|
||||
"=r"(data[84]), "=r"(data[85]), "=r"(data[86]), "=r"(data[87]),
|
||||
"=r"(data[88]), "=r"(data[89]), "=r"(data[90]), "=r"(data[91]),
|
||||
"=r"(data[92]), "=r"(data[93]), "=r"(data[94]), "=r"(data[95]),
|
||||
"=r"(data[96]), "=r"(data[97]), "=r"(data[98]), "=r"(data[99]),
|
||||
"=r"(data[100]), "=r"(data[101]), "=r"(data[102]), "=r"(data[103]),
|
||||
"=r"(data[104]), "=r"(data[105]), "=r"(data[106]), "=r"(data[107]),
|
||||
"=r"(data[108]), "=r"(data[109]), "=r"(data[110]), "=r"(data[111]),
|
||||
"=r"(data[112]), "=r"(data[113]), "=r"(data[114]), "=r"(data[115]),
|
||||
"=r"(data[116]), "=r"(data[117]), "=r"(data[118]), "=r"(data[119]),
|
||||
"=r"(data[120]), "=r"(data[121]), "=r"(data[122]), "=r"(data[123]),
|
||||
"=r"(data[124]), "=r"(data[125]), "=r"(data[126]), "=r"(data[127]),
|
||||
"=f"(redval)
|
||||
: "r"(tmem_start),
|
||||
"C"(USE_MAX ? s_max : s_min),
|
||||
"C"(USE_ABS ? s_abs : s_empty),
|
||||
"C"(USE_NAN ? s_nan : s_empty)
|
||||
);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
// Load from tensor memory, 16 data path lanes, 128-bit pattern, repeated N times. (https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-instructions-tcgen05-ld)
|
||||
template <int kNumReplications>
|
||||
__device__ __forceinline__
|
||||
void tmem_ld_16dp128bNx(uint32_t tmem_start, void* data_) {
|
||||
uint32_t* data = (uint32_t*)data_;
|
||||
static_assert(kNumReplications == 1 || kNumReplications == 2 || kNumReplications == 4 || kNumReplications == 8 || kNumReplications == 16 || kNumReplications == 32 || kNumReplications == 64, "Invalid kNumReplications");
|
||||
#ifndef __VSCODE_IDE__
|
||||
[&]<size_t... Is>(cute::index_sequence<Is...>) {
|
||||
if constexpr (kNumReplications == 1) {
|
||||
cute::SM100_TMEM_LOAD_16dp128b1x::copy(tmem_start, data[Is]...);
|
||||
} else if constexpr (kNumReplications == 2) {
|
||||
cute::SM100_TMEM_LOAD_16dp128b2x::copy(tmem_start, data[Is]...);
|
||||
} else if constexpr (kNumReplications == 4) {
|
||||
cute::SM100_TMEM_LOAD_16dp128b4x::copy(tmem_start, data[Is]...);
|
||||
} else if constexpr (kNumReplications == 8) {
|
||||
cute::SM100_TMEM_LOAD_16dp128b8x::copy(tmem_start, data[Is]...);
|
||||
} else if constexpr (kNumReplications == 16) {
|
||||
cute::SM100_TMEM_LOAD_16dp128b16x::copy(tmem_start, data[Is]...);
|
||||
} else if constexpr (kNumReplications == 32) {
|
||||
cute::SM100_TMEM_LOAD_16dp128b32x::copy(tmem_start, data[Is]...);
|
||||
} else if constexpr (kNumReplications == 64) {
|
||||
cute::SM100_TMEM_LOAD_16dp128b64x::copy(tmem_start, data[Is]...);
|
||||
}
|
||||
}(cute::make_index_sequence<kNumReplications*2>{});
|
||||
#endif
|
||||
}
|
||||
|
||||
// Load from tensor memory, 16 data path lanes, 256-bit pattern, repeated N times. (https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-instructions-tcgen05-ld)
|
||||
template <int kNumReplications>
|
||||
__device__ __forceinline__
|
||||
void tmem_ld_16dp256bNx(uint32_t tmem_start, void* data_) {
|
||||
uint32_t* data = (uint32_t*)data_;
|
||||
static_assert(kNumReplications == 1 || kNumReplications == 2 || kNumReplications == 4 || kNumReplications == 8 || kNumReplications == 16 || kNumReplications == 32, "Invalid kNumReplications");
|
||||
#ifndef __VSCODE_IDE__
|
||||
[&]<size_t... Is>(cute::index_sequence<Is...>) {
|
||||
if constexpr (kNumReplications == 1) {
|
||||
cute::SM100_TMEM_LOAD_16dp256b1x::copy(tmem_start, data[Is]...);
|
||||
} else if constexpr (kNumReplications == 2) {
|
||||
cute::SM100_TMEM_LOAD_16dp256b2x::copy(tmem_start, data[Is]...);
|
||||
} else if constexpr (kNumReplications == 4) {
|
||||
cute::SM100_TMEM_LOAD_16dp256b4x::copy(tmem_start, data[Is]...);
|
||||
} else if constexpr (kNumReplications == 8) {
|
||||
cute::SM100_TMEM_LOAD_16dp256b8x::copy(tmem_start, data[Is]...);
|
||||
} else if constexpr (kNumReplications == 16) {
|
||||
cute::SM100_TMEM_LOAD_16dp256b16x::copy(tmem_start, data[Is]...);
|
||||
} else if constexpr (kNumReplications == 32) {
|
||||
cute::SM100_TMEM_LOAD_16dp256b32x::copy(tmem_start, data[Is]...);
|
||||
}
|
||||
}(cute::make_index_sequence<kNumReplications*4>{});
|
||||
#endif
|
||||
}
|
||||
|
||||
// Store into tensor memory, 32 data path lanes, 32-bit pattern, repeated N times. (https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-instructions-tcgen05-st)
|
||||
template <int kNumElements>
|
||||
__device__ __forceinline__
|
||||
void tmem_st_32dp32bNx(uint32_t tmem_start, void const* data_) {
|
||||
uint32_t const* data = (uint32_t const*)data_;
|
||||
static_assert(kNumElements == 1 || kNumElements == 2 || kNumElements == 4 || kNumElements == 8 || kNumElements == 16 || kNumElements == 32 || kNumElements == 64 || kNumElements == 128, "Invalid kNumElements");
|
||||
#ifndef __VSCODE_IDE__
|
||||
[&]<size_t... Is>(cute::index_sequence<Is...>) {
|
||||
if constexpr (kNumElements == 1) {
|
||||
cute::SM100_TMEM_STORE_32dp32b1x::copy(data[Is]..., tmem_start);
|
||||
} else if constexpr (kNumElements == 2) {
|
||||
cute::SM100_TMEM_STORE_32dp32b2x::copy(data[Is]..., tmem_start);
|
||||
} else if constexpr (kNumElements == 4) {
|
||||
cute::SM100_TMEM_STORE_32dp32b4x::copy(data[Is]..., tmem_start);
|
||||
} else if constexpr (kNumElements == 8) {
|
||||
cute::SM100_TMEM_STORE_32dp32b8x::copy(data[Is]..., tmem_start);
|
||||
} else if constexpr (kNumElements == 16) {
|
||||
cute::SM100_TMEM_STORE_32dp32b16x::copy(data[Is]..., tmem_start);
|
||||
} else if constexpr (kNumElements == 32) {
|
||||
cute::SM100_TMEM_STORE_32dp32b32x::copy(data[Is]..., tmem_start);
|
||||
} else if constexpr (kNumElements == 64) {
|
||||
cute::SM100_TMEM_STORE_32dp32b64x::copy(data[Is]..., tmem_start);
|
||||
} else if constexpr (kNumElements == 128) {
|
||||
cute::SM100_TMEM_STORE_32dp32b128x::copy(data[Is]..., tmem_start);
|
||||
}
|
||||
}(cute::make_index_sequence<kNumElements>{});
|
||||
#endif
|
||||
}
|
||||
|
||||
// Decompose a non-power-of-2 replication count into a sequence of power-of-2 calls.
|
||||
//
|
||||
// This helper splits kNumReplications into its constituent powers of 2 (from highest to lowest)
|
||||
// and issues one call to tmem_ld_st_fn.template operator()<power>() for each, automatically
|
||||
// advancing the tmem address by kNumElemsPerUnit columns and the data pointer by kNumElemsPerUnit
|
||||
// uint32_t elements per unit.
|
||||
//
|
||||
// Template parameters:
|
||||
// kNumReplications — total number of replications (may be non-power-of-2)
|
||||
// kNumElemsPerUnit — number of tmem columns (and uint32_t data elements) consumed per unit
|
||||
// (e.g. 1 for 32dp32b, 2 for 16dp128b, 4 for 16dp256b)
|
||||
// F — a callable with a template operator(): f.template operator()<N>(uint32_t, void*)
|
||||
//
|
||||
// Usage example (with a template lambda):
|
||||
// tmem_ld_st_decomposed<6, 1>(tmem_start, data_ptr,
|
||||
// []<int N>(uint32_t ts, void* d) { tmem_ld_32dp32bNx<N>(ts, d); });
|
||||
// // Equivalent to:
|
||||
// // tmem_ld_32dp32bNx<4>(tmem_start, data_ptr);
|
||||
// // tmem_ld_32dp32bNx<2>(tmem_start + 4, (void*)((uint32_t*)data_ptr + 4));
|
||||
template<int kNumReplications, int kNumElemsPerUnit, typename F>
|
||||
__device__ __forceinline__
|
||||
void tmem_ld_st_decomposed(uint32_t tmem_start, void* data, F&& tmem_ld_st_fn) {
|
||||
if constexpr (kNumReplications == 0) {
|
||||
// Base case: nothing to do
|
||||
return;
|
||||
} else {
|
||||
// Largest power-of-2 <= kNumReplications
|
||||
constexpr int kHighBit = 1 << (31 - __builtin_clz(kNumReplications));
|
||||
tmem_ld_st_fn.template operator()<kHighBit>(tmem_start, data);
|
||||
// Recurse on the remainder
|
||||
constexpr int kRemaining = kNumReplications - kHighBit;
|
||||
if constexpr (kRemaining > 0) {
|
||||
tmem_ld_st_decomposed<kRemaining, kNumElemsPerUnit>(
|
||||
tmem_start + kHighBit * kNumElemsPerUnit,
|
||||
(void*)((uint32_t*)data + kHighBit * kNumElemsPerUnit),
|
||||
tmem_ld_st_fn
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
+55
@@ -0,0 +1,55 @@
|
||||
#pragma once
|
||||
|
||||
#include "kerutils/device/cuda/common.h"
|
||||
#include "kerutils/device/cuda/sm80/intrinsics.cuh"
|
||||
|
||||
namespace kerutils {
|
||||
|
||||
// Retrieve the value of `%smid` and check its range
|
||||
CUTE_DEVICE
|
||||
uint32_t get_sm_id_with_range_check(uint32_t num_physical_sms) {
|
||||
uint32_t sm_id = get_sm_id();
|
||||
if (!(sm_id < num_physical_sms)) {
|
||||
trap();
|
||||
}
|
||||
return sm_id;
|
||||
}
|
||||
|
||||
#ifndef KU_TRAP_ONLY_DEVICE_ASSERT
|
||||
#define KU_TRAP_ONLY_DEVICE_ASSERT(cond) \
|
||||
do { \
|
||||
if (not (cond)) \
|
||||
asm("trap;"); \
|
||||
} while (0)
|
||||
#endif
|
||||
|
||||
// Construct a `float2` from a single `float` by duplicating the value
|
||||
CUTE_DEVICE
|
||||
float2 float2float2(const float &x) {
|
||||
return float2 {x, x};
|
||||
}
|
||||
|
||||
CUTE_DEVICE
|
||||
void st_shared(void* ptr, __int128_t val) {
|
||||
asm volatile("st.shared.b128 [%0], %1;" :: "r"(cute::cast_smem_ptr_to_uint(ptr)), "q"(val));
|
||||
}
|
||||
|
||||
CUTE_DEVICE
|
||||
void st_shared(void* ptr, float4 val) {
|
||||
st_shared(ptr, *(__int128_t*)&val);
|
||||
}
|
||||
|
||||
CUTE_DEVICE
|
||||
__int128_t ld_shared(const void* ptr) {
|
||||
__int128_t val;
|
||||
asm volatile("ld.shared.b128 %0, [%1];" : "=q"(val) : "r"(cute::cast_smem_ptr_to_uint(ptr)));
|
||||
return val;
|
||||
}
|
||||
|
||||
CUTE_DEVICE
|
||||
float4 ld_shared_float4(const void* ptr) {
|
||||
__int128_t temp = ld_shared(ptr);
|
||||
return *(float4*)&temp;
|
||||
}
|
||||
|
||||
}
|
||||
+160
@@ -0,0 +1,160 @@
|
||||
#pragma once
|
||||
|
||||
#include "kerutils/device/cuda/common.h"
|
||||
|
||||
namespace kerutils {
|
||||
|
||||
// cp.async.cg (cache global) with prefetch and predicate (https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-cp-async)
|
||||
template<PrefetchSize PREFETCH_SIZE=PrefetchSize::B128>
|
||||
CUTE_DEVICE
|
||||
void cp_async_cacheglobal(const void* src, void* dst, bool pred=true) {
|
||||
uint32_t dst_addr = cute::cast_smem_ptr_to_uint(dst);
|
||||
if constexpr (PREFETCH_SIZE == PrefetchSize::B64) {
|
||||
asm volatile("cp.async.cg.shared.global.L2::64B [%0], [%1], 16, %2;\n"
|
||||
:: "r"(dst_addr),
|
||||
"l"(src),
|
||||
"r"(pred?16:0));
|
||||
} else if constexpr (PREFETCH_SIZE == PrefetchSize::B128) {
|
||||
asm volatile("cp.async.cg.shared.global.L2::128B [%0], [%1], 16, %2;\n"
|
||||
:: "r"(dst_addr),
|
||||
"l"(src),
|
||||
"r"(pred?16:0));
|
||||
} else if constexpr (PREFETCH_SIZE == PrefetchSize::B256) {
|
||||
asm volatile("cp.async.cg.shared.global.L2::256B [%0], [%1], 16, %2;\n"
|
||||
:: "r"(dst_addr),
|
||||
"l"(src),
|
||||
"r"(pred?16:0));
|
||||
} else {
|
||||
static_assert(PREFETCH_SIZE == PrefetchSize::B64 ||
|
||||
PREFETCH_SIZE == PrefetchSize::B128 ||
|
||||
PREFETCH_SIZE == PrefetchSize::B256,
|
||||
"Unsupported prefetch size for cp_async_cacheglobal.");
|
||||
}
|
||||
}
|
||||
|
||||
// Create fraction-based cache policy (https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-createpolicy)
|
||||
template<CacheHint PRIMARY_PRIORITY, CacheHint SECONDARY_PRIORITY>
|
||||
CUTE_DEVICE
|
||||
int64_t create_fraction_based_cache_policy(float fraction = 1.0f) {
|
||||
int64_t result;
|
||||
#define EMIT(PRIMARY_PRIORITY_STR, SECONDARY_PRIORITY_STR) \
|
||||
asm volatile( \
|
||||
"createpolicy.fractional.L2::" PRIMARY_PRIORITY_STR ".L2::" SECONDARY_PRIORITY_STR ".b64 %0, %1;\n" \
|
||||
: "=l"(result) \
|
||||
: "f"(fraction) \
|
||||
);
|
||||
#define EMIT2(PRIMARY_PRIORITY_STR) \
|
||||
{ \
|
||||
if constexpr (SECONDARY_PRIORITY == CacheHint::EVICT_FIRST) { \
|
||||
EMIT(PRIMARY_PRIORITY_STR, "evict_first") \
|
||||
} else if constexpr (SECONDARY_PRIORITY == CacheHint::EVICT_UNCHANGED) { \
|
||||
EMIT(PRIMARY_PRIORITY_STR, "evict_unchanged") \
|
||||
} else { \
|
||||
static_assert(SECONDARY_PRIORITY == CacheHint::EVICT_FIRST || \
|
||||
SECONDARY_PRIORITY == CacheHint::EVICT_UNCHANGED, \
|
||||
"Unsupported secondary cache hint for create_fraction_based_cache_policy."); \
|
||||
} \
|
||||
}
|
||||
if constexpr (PRIMARY_PRIORITY == CacheHint::EVICT_FIRST) {
|
||||
EMIT2("evict_first");
|
||||
} else if constexpr (PRIMARY_PRIORITY == CacheHint::EVICT_NORMAL) {
|
||||
EMIT2("evict_normal");
|
||||
} else if constexpr (PRIMARY_PRIORITY == CacheHint::EVICT_LAST) {
|
||||
EMIT2("evict_last");
|
||||
} else if constexpr (PRIMARY_PRIORITY == CacheHint::EVICT_UNCHANGED) {
|
||||
EMIT2("evict_unchanged");
|
||||
} else {
|
||||
static_assert(PRIMARY_PRIORITY == CacheHint::EVICT_FIRST ||
|
||||
PRIMARY_PRIORITY == CacheHint::EVICT_NORMAL ||
|
||||
PRIMARY_PRIORITY == CacheHint::EVICT_LAST ||
|
||||
PRIMARY_PRIORITY == CacheHint::EVICT_UNCHANGED,
|
||||
"Unsupported primary cache hint for create_fraction_based_cache_policy.");
|
||||
}
|
||||
#undef EMIT
|
||||
#undef EMIT2
|
||||
return result;
|
||||
}
|
||||
|
||||
// Create a simple cache policy (equivalent to create_fraction_based_cache_policy(1.0f))
|
||||
// The same as cute::TMA::CacheHintSmXX
|
||||
template<CacheHint CACHE_HINT>
|
||||
CUTE_DEVICE
|
||||
constexpr int64_t create_simple_cache_policy() {
|
||||
if constexpr (CACHE_HINT == CacheHint::EVICT_FIRST) {
|
||||
return 0x12F0000000000000; // Result of createpolicy.fractional.L2::evict_first.b64
|
||||
} else if constexpr (CACHE_HINT == CacheHint::EVICT_NORMAL) {
|
||||
return 0x1000000000000000; // Copied from CuTe. Unsure about the exact meaning. (TODO Change to 0x16F0000000000000?)
|
||||
} else if constexpr (CACHE_HINT == CacheHint::EVICT_LAST) {
|
||||
return 0x14F0000000000000; // Result of createpolicy.fractional.L2::evict_last.b64
|
||||
} else {
|
||||
static_assert(CACHE_HINT == CacheHint::EVICT_FIRST ||
|
||||
CACHE_HINT == CacheHint::EVICT_NORMAL ||
|
||||
CACHE_HINT == CacheHint::EVICT_LAST,
|
||||
"Unsupported cache hint for create_simple_cache_policy.");
|
||||
}
|
||||
}
|
||||
|
||||
// AtomicAdd (https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-red)
|
||||
CUTE_DEVICE
|
||||
void atomicadd_f32_with_policy_and_pred(void* global_addr, const float &data, int64_t cache_policy, uint32_t pred = true) {
|
||||
asm volatile(
|
||||
"{\n\t"
|
||||
".reg .pred p;\n\t"
|
||||
"setp.eq.u32 p, %3, 1;\n\t"
|
||||
"@p red.relaxed.gpu.global.add.L2::cache_hint.f32 [%1], %0, %2; \n\t"
|
||||
"}"
|
||||
:
|
||||
: "f"(data),
|
||||
"l"((int64_t)global_addr), "l"(cache_policy), "r"(pred)
|
||||
);
|
||||
}
|
||||
|
||||
// Get the id of the current SM
|
||||
// About %smid (https://docs.nvidia.com/cuda/parallel-thread-execution/#special-registers-smid): PTX document says that %smid ranges from 0 to %nsmid-1, while "The SM identifier numbering is not guaranteed to be contiguous, so %nsmid may be larger than the physical number of SMs in the device.". However, result shows that, at least for sm90 and sm100, %nsmid is the number of physical SMs - 1. For the sake of safety, I recommend you to check the return of get_sm_id manually or call `get_sm_id_with_range_check()` defined in `device/sm80/helpers.cuh`.
|
||||
// Besides, PTX document also says that this number may change due to preemption, but currently this never happens according to [DATEN GELÖSCHT]
|
||||
CUTE_DEVICE
|
||||
uint32_t get_sm_id() {
|
||||
uint32_t ret;
|
||||
asm volatile("mov.u32 %0, %%smid;\n" : "=r"(ret));
|
||||
return ret;
|
||||
}
|
||||
|
||||
// trap (https://docs.nvidia.com/cuda/parallel-thread-execution/#miscellaneous-instructions-trap)
|
||||
CUTE_DEVICE
|
||||
void trap() {
|
||||
asm volatile("trap;\n");
|
||||
}
|
||||
|
||||
// LDG.128 or LDG.128 with non-coherent cache (https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-ld)
|
||||
// We use macro instead of function here, since we need a multi-level recursive dispatch based on template parameters if using function
|
||||
// NC_STR should be either "" or ".nc"
|
||||
// L1_CACHE_HINT_STR should be either "evict_first", "evict_normal", "evict_last", "evict_unchanged", or "no_allocate"
|
||||
// L2_PREFETCH_SIZE_STR should be either "64B", "128B", or "256B"
|
||||
// L2 cache hint is not supported since it's only supported for LDG.256
|
||||
#define KU_LDG_128(global_addr, result, NC_STR, L1_CACHE_HINT_STR, L2_PREFETCH_SIZE_STR) \
|
||||
{ \
|
||||
static_assert(std::is_pointer_v<decltype(global_addr)> || std::is_array_v<decltype(global_addr)>, "`global_addr` must be a pointer"); \
|
||||
static_assert(std::is_pointer_v<decltype(result)> || std::is_array_v<decltype(result)>, "`result` must be a pointer"); \
|
||||
uint64_t* result_as_uint64_ptr = (uint64_t*)(result); \
|
||||
asm volatile( \
|
||||
"ld.global" NC_STR ".L1::" L1_CACHE_HINT_STR ".L2::" L2_PREFETCH_SIZE_STR ".v2.u64 {%0, %1}, [%2];\n" \
|
||||
: "=l"(result_as_uint64_ptr[0]), "=l"(result_as_uint64_ptr[1]) \
|
||||
: "l"(global_addr) \
|
||||
); \
|
||||
}
|
||||
|
||||
// STG.128 (https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-st)
|
||||
// L2_CACHE_HINT_STR should be ".L2::evict_XXX" (only available on sm100+) or ""
|
||||
#define KU_STG_128(global_addr, src, L1_CACHE_HINT_STR, L2_CACHE_HINT_STR) \
|
||||
{ \
|
||||
static_assert(std::is_pointer_v<decltype(global_addr)> || std::is_array_v<decltype(global_addr)>, "`global_addr` must be a pointer"); \
|
||||
static_assert(std::is_pointer_v<decltype(src)> || std::is_array_v<decltype(src)>, "`src` must be a pointer"); \
|
||||
uint64_t const* src_as_uint64_ptr = (uint64_t const*)(src); \
|
||||
asm volatile( \
|
||||
"st.global.L1::" L1_CACHE_HINT_STR L2_CACHE_HINT_STR ".v2.u64 [%0], {%1, %2};\n" \
|
||||
: \
|
||||
: "l"(global_addr), "l"(src_as_uint64_ptr[0]), "l"(src_as_uint64_ptr[1]) \
|
||||
); \
|
||||
}
|
||||
|
||||
}
|
||||
+107
@@ -0,0 +1,107 @@
|
||||
#pragma once
|
||||
|
||||
#include "kerutils/device/cuda/common.h"
|
||||
|
||||
namespace kerutils {
|
||||
|
||||
// st.async (https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-st-async)
|
||||
template<typename T>
|
||||
CUTE_DEVICE
|
||||
static void st_async(void* dst_ptr, const T& data, transac_bar_t &mbar) {
|
||||
static_assert(sizeof(T) == 16, "Data type must be 16 bytes (128 bits) for st_async.");
|
||||
long2 data_long2 = *reinterpret_cast<const long2*>(&data);
|
||||
uint32_t dst_addr = cute::cast_smem_ptr_to_uint(dst_ptr);
|
||||
uint32_t mbar_addr = cute::cast_smem_ptr_to_uint(&mbar);
|
||||
asm volatile (
|
||||
"st.async.weak.shared::cluster.mbarrier::complete_tx::bytes.v2.s64 [%0], {%1, %2}, [%3]; \n"
|
||||
:
|
||||
: "r"(dst_addr), "l"(data_long2.x), "l"(data_long2.y), "r"(mbar_addr)
|
||||
);
|
||||
}
|
||||
|
||||
static constexpr int PEER_ADDR_MASK = 16777216;
|
||||
|
||||
// Given an address in the current CTA, return the corresponding address in the peer CTA
|
||||
template<typename T>
|
||||
CUTE_DEVICE
|
||||
T* get_peer_addr(const T* p) {
|
||||
return (T*)((int64_t)(p) ^ PEER_ADDR_MASK);
|
||||
}
|
||||
|
||||
// Given an address in the current CTA, return the corresponding address in the peer CTA (if the current CTA_id%2 == 1) or the address itself (if CTA_id%2 == 0)
|
||||
template<typename T>
|
||||
CUTE_DEVICE
|
||||
T* get_cta0_addr(const T* p) {
|
||||
constexpr int CTA0_ADDR_MASK = 0xFEFFFFFF;
|
||||
return (T*)((int64_t)(p) & CTA0_ADDR_MASK);
|
||||
}
|
||||
|
||||
// TMA bulk reduce add (cp.reduce.async.bulk), shared to global, float32, add. (https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-cp-reduce-async-bulk)
|
||||
CUTE_DEVICE
|
||||
void tma_bulk_reduce_add(void const* src_ptr, void* dst_ptr, int32_t store_bytes) {
|
||||
uint32_t smem_int_ptr = cute::cast_smem_ptr_to_uint(src_ptr);
|
||||
asm volatile("cp.reduce.async.bulk.global.shared::cta.bulk_group.add.f32 [%0], [%1], %2;\n"
|
||||
:
|
||||
: "l"(dst_ptr), "r"(smem_int_ptr), "r"(store_bytes)
|
||||
: "memory");
|
||||
}
|
||||
|
||||
// Cluster barrier arrive with .release modifier. (https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-barrier-cluster)
|
||||
CUTE_DEVICE
|
||||
void barrier_cluster_arrive_release() {
|
||||
asm volatile("barrier.cluster.arrive.release;" : : : "memory");
|
||||
}
|
||||
|
||||
// Cluster barrier arrive with .relaxed modifier. (https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-barrier-cluster)
|
||||
CUTE_DEVICE
|
||||
void barrier_cluster_arrive_relaxed() {
|
||||
asm volatile("barrier.cluster.arrive.relaxed;" : : :);
|
||||
}
|
||||
|
||||
// Cluster barrier wait with .acquire modifier. (https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-barrier-cluster)
|
||||
CUTE_DEVICE
|
||||
void barrier_cluster_wait_acquire() {
|
||||
asm volatile("barrier.cluster.wait.acquire;" : : : "memory");
|
||||
}
|
||||
|
||||
// mbarrier.arrive with .relaxed.cluster qualifier (https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-mbarrier-arrive)
|
||||
CUTE_DEVICE
|
||||
void mbarrier_arrive_relaxed_cluster(transac_bar_t &mbar) {
|
||||
uint32_t smem_addr = cute::cast_smem_ptr_to_uint(&mbar);
|
||||
asm volatile(
|
||||
"{\n\t"
|
||||
"mbarrier.arrive.relaxed.cluster.shared::cta.b64 _, [%0];\n\t"
|
||||
"}"
|
||||
:
|
||||
: "r"(smem_addr));
|
||||
}
|
||||
|
||||
// AtomicAdd with v4.f32 type (https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-red)
|
||||
CUTE_DEVICE
|
||||
void atomicadd_f32x4_with_policy_and_pred(void* global_addr, const float4 &data, int64_t cache_policy, uint32_t pred = true) {
|
||||
asm volatile(
|
||||
"{\n\t"
|
||||
".reg .pred p;\n\t"
|
||||
"setp.eq.u32 p, %6, 1;\n\t"
|
||||
"@p red.relaxed.gpu.global.add.L2::cache_hint.v4.f32 [%4], {%0, %1, %2, %3}, %5; \n\t"
|
||||
"}"
|
||||
:
|
||||
: "f"(data.x), "f"(data.y), "f"(data.z), "f"(data.w),
|
||||
"l"((int64_t)global_addr), "l"(cache_policy), "r"(pred)
|
||||
);
|
||||
}
|
||||
|
||||
// cp.async.bulk, from .shared::cta to .shared::cluster (https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-cp-async-bulk)
|
||||
CUTE_DEVICE
|
||||
void cp_async_bulk_shared_cta_to_shared_cluster(void* dst_ptr, const void* src_ptr, int32_t load_bytes, transac_bar_t &mbar) {
|
||||
uint32_t dst_smem_addr = cute::cast_smem_ptr_to_uint(dst_ptr);
|
||||
uint32_t src_smem_addr = cute::cast_smem_ptr_to_uint(src_ptr);
|
||||
uint32_t mbar_addr = cute::cast_smem_ptr_to_uint(&mbar);
|
||||
asm volatile(
|
||||
"cp.async.bulk.shared::cluster.shared::cta.mbarrier::complete_tx::bytes [%0], [%1], %2, [%3]; \n"
|
||||
:
|
||||
: "r"(dst_smem_addr), "r"(src_smem_addr), "r"(load_bytes), "r"(mbar_addr)
|
||||
);
|
||||
}
|
||||
|
||||
}
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
#pragma once
|
||||
|
||||
#include "kerutils/common/common.h"
|
||||
|
||||
#ifdef KERUTILS_IS_BUILD_ON_CUDA
|
||||
#include "cuda/common.h"
|
||||
#include "cuda/sm80/intrinsics.cuh"
|
||||
#include "cuda/sm80/helpers.cuh"
|
||||
#include "cuda/sm90/intrinsics.cuh"
|
||||
#include "cuda/sm100/intrinsics.cuh"
|
||||
#endif
|
||||
Vendored
+282
@@ -0,0 +1,282 @@
|
||||
#pragma once
|
||||
|
||||
#include <cstdio>
|
||||
#include <exception>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <sstream>
|
||||
#include <vector>
|
||||
|
||||
#ifdef KERUTILS_IS_BUILD_ON_CUDA
|
||||
#include <cuda_runtime_api.h>
|
||||
#include <cudaTypedefs.h>
|
||||
#endif
|
||||
|
||||
#include "kerutils/common/common.h"
|
||||
|
||||
namespace kerutils {
|
||||
|
||||
class KUException final : public std::exception {
|
||||
std::string message = {};
|
||||
|
||||
public:
|
||||
template<typename... Args>
|
||||
explicit KUException(const char *name, const char* file, const int line, Args&&... args) {
|
||||
std::ostringstream oss;
|
||||
|
||||
oss << name << " error (" << file << ":" << line << "): ";
|
||||
(oss << ... << args);
|
||||
message = oss.str();
|
||||
}
|
||||
|
||||
const char *what() const noexcept override {
|
||||
return message.c_str();
|
||||
}
|
||||
};
|
||||
|
||||
#define THROW_KU_EXCEPTION(name, ...) \
|
||||
throw kerutils::KUException(name, __FILE__, __LINE__, __VA_ARGS__)
|
||||
|
||||
// This `KU_ASSERT` is triggered no matter if the code is compiled with `-DNDEBUG` or not.
|
||||
#define KU_ASSERT(cond, ...) \
|
||||
do { \
|
||||
if (not (cond)) { \
|
||||
char _ku_buf[1024]; \
|
||||
int _ku_len = snprintf(_ku_buf, sizeof(_ku_buf), \
|
||||
"Assertion `%s` failed (%s:%d)", #cond, __FILE__, __LINE__); \
|
||||
__VA_OPT__( \
|
||||
_ku_len += snprintf(_ku_buf + _ku_len, sizeof(_ku_buf) - _ku_len, \
|
||||
": " __VA_ARGS__); \
|
||||
) \
|
||||
fprintf(stderr, "%s\n", _ku_buf); \
|
||||
THROW_KU_EXCEPTION("Assertion", _ku_buf); \
|
||||
} \
|
||||
} while(0)
|
||||
|
||||
#ifdef KERUTILS_IS_BUILD_ON_CUDA
|
||||
|
||||
#define KU_CUDA_CHECK(call) \
|
||||
do { \
|
||||
cudaError_t status_ = (call); \
|
||||
if (status_ != cudaSuccess) { \
|
||||
char _ku_buf[1024]; \
|
||||
snprintf(_ku_buf, sizeof(_ku_buf), "CUDA error (%s:%d): %s", __FILE__, __LINE__, cudaGetErrorString(status_)); \
|
||||
fprintf(stderr, "%s\n", _ku_buf); \
|
||||
THROW_KU_EXCEPTION("CUDA", _ku_buf); \
|
||||
} \
|
||||
} while(0)
|
||||
|
||||
#define KU_CUTLASS_CHECK(call) \
|
||||
do { \
|
||||
cutlass::Status status_ = (call); \
|
||||
if (status_ != cutlass::Status::kSuccess) { \
|
||||
char _ku_buf[1024]; \
|
||||
snprintf(_ku_buf, sizeof(_ku_buf), "CUTLASS error (%s:%d): %d", __FILE__, __LINE__, static_cast<int>(status_)); \
|
||||
fprintf(stderr, "%s\n", _ku_buf); \
|
||||
THROW_KU_EXCEPTION("CUTLASS", _ku_buf); \
|
||||
} \
|
||||
} while(0)
|
||||
|
||||
#define KU_CHECK_KERNEL_LAUNCH() KU_CUDA_CHECK(cudaGetLastError())
|
||||
|
||||
template<typename T>
|
||||
inline __host__ __device__ constexpr T ceil_div(const T &a, const T &b) {
|
||||
return (a + b - 1) / b;
|
||||
}
|
||||
|
||||
template<typename T>
|
||||
inline __host__ __device__ constexpr T ceil(const T &a, const T &b) {
|
||||
return (a + b - 1) / b * b;
|
||||
}
|
||||
|
||||
template<typename T, T LOWER_BOUND = 1>
|
||||
inline __host__ __device__ constexpr T find_next_power_of_2(const T& x) {
|
||||
if (x <= LOWER_BOUND)
|
||||
return LOWER_BOUND;
|
||||
return find_next_power_of_2<T, LOWER_BOUND*2>(x);
|
||||
}
|
||||
|
||||
// A wrapper for make_tensor_map
|
||||
static inline CUtensorMap make_tensor_map(
|
||||
const std::vector<uint64_t> &size,
|
||||
const std::vector<uint64_t> &strides, // PAY ATTENTION: In BYTES
|
||||
const std::vector<uint32_t> &box_size,
|
||||
void* global_ptr,
|
||||
CUtensorMapDataType data_type,
|
||||
CUtensorMapSwizzle swizzle_mode,
|
||||
CUtensorMapL2promotion l2_promotion,
|
||||
CUtensorMapInterleave interleave_mode = CUtensorMapInterleave::CU_TENSOR_MAP_INTERLEAVE_NONE,
|
||||
CUtensorMapFloatOOBfill oob_fill = CUtensorMapFloatOOBfill::CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE,
|
||||
const std::vector<uint32_t> &element_strides_ = {}
|
||||
) {
|
||||
int dim = size.size();
|
||||
KU_ASSERT(dim >= 1);
|
||||
|
||||
std::vector<uint32_t> element_strides;
|
||||
if (element_strides_.empty()) {
|
||||
for (int i = 0; i < dim; ++i)
|
||||
element_strides.push_back(1);
|
||||
} else {
|
||||
element_strides = element_strides_;
|
||||
}
|
||||
KU_ASSERT(strides.size() == (uint32_t)dim-1 && box_size.size() == (uint32_t)dim && element_strides.size() == (uint32_t)dim);
|
||||
|
||||
auto call_cuTensorMapEncodeTiled = [&]<typename... Args>(Args... args) {
|
||||
cudaDriverEntryPointQueryResult cuda_status;
|
||||
void* pfn = nullptr;
|
||||
#if (__CUDACC_VER_MAJOR__ > 12)
|
||||
KU_CUDA_CHECK(cudaGetDriverEntryPointByVersion(
|
||||
"cuTensorMapEncodeTiled",
|
||||
&pfn, 12000,
|
||||
cudaEnableDefault,
|
||||
&cuda_status));
|
||||
#else
|
||||
KU_CUDA_CHECK(cudaGetDriverEntryPoint(
|
||||
"cuTensorMapEncodeTiled",
|
||||
&pfn,
|
||||
cudaEnableDefault,
|
||||
&cuda_status));
|
||||
#endif
|
||||
if (cuda_status != cudaDriverEntryPointSuccess) {
|
||||
KU_ASSERT(false, "Failed to load `cuTensorMapEncodeTiled`. cuda_status = %d", cuda_status);
|
||||
}
|
||||
return reinterpret_cast<decltype(&cuTensorMapEncodeTiled)>(pfn)(args...); \
|
||||
};
|
||||
|
||||
CUtensorMap result;
|
||||
CUresult ret_code = call_cuTensorMapEncodeTiled(
|
||||
&result,
|
||||
data_type,
|
||||
dim,
|
||||
global_ptr,
|
||||
size.data(),
|
||||
strides.data(),
|
||||
box_size.data(),
|
||||
element_strides.data(),
|
||||
interleave_mode,
|
||||
swizzle_mode,
|
||||
l2_promotion,
|
||||
oob_fill
|
||||
);
|
||||
if (ret_code != CUresult::CUDA_SUCCESS) {
|
||||
auto print_vector = [&](auto t, const char* fmt, const char end='\n') {
|
||||
for (auto elem : t) {
|
||||
printf(fmt, elem);
|
||||
}
|
||||
printf("%c", end);
|
||||
};
|
||||
fprintf(stderr, "Failed to create tensormap\n");
|
||||
fprintf(stderr, "Dim: %d\n", dim);
|
||||
printf("size: "); print_vector(size, "%lu ");
|
||||
printf("strides: "); print_vector(strides, "%lu ");
|
||||
printf("box_size: "); print_vector(box_size, "%u ");
|
||||
printf("element_strides: "); print_vector(element_strides, "%u ");
|
||||
printf("global ptr: 0x%lx\n", (int64_t)global_ptr);
|
||||
printf("data_type: %d\n", (int)data_type);
|
||||
printf("swizzle_mode: %d\n", (int)swizzle_mode);
|
||||
printf("l2_promotion: %d\n", (int)l2_promotion);
|
||||
printf("interleave_mode: %d\n", (int)interleave_mode);
|
||||
printf("oob_fill: %d\n", (int)oob_fill);
|
||||
KU_ASSERT(false);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
// Given strides (in number of elements), this function converts their datatype in uint64_t and then multiplies by elem_size
|
||||
template<typename T>
|
||||
static inline std::vector<uint64_t> make_stride_helper(const std::vector<T> &strides_in_elems, size_t elem_size) {
|
||||
std::vector<uint64_t> res;
|
||||
for (auto stride : strides_in_elems) {
|
||||
res.push_back(((uint64_t)stride) * elem_size);
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
struct KernelLaunchConfig {
|
||||
dim3 grid;
|
||||
dim3 block;
|
||||
size_t dynamic_smem;
|
||||
cudaStream_t stream;
|
||||
dim3 cluster{1, 1, 1};
|
||||
bool use_pdl{false};
|
||||
bool cooperative{false};
|
||||
};
|
||||
|
||||
namespace detail {
|
||||
|
||||
template <class Arg>
|
||||
void* kernel_arg_ptr(Arg&& arg) {
|
||||
return const_cast<void*>(reinterpret_cast<const void*>(std::addressof(arg)));
|
||||
}
|
||||
|
||||
} // namespace detail
|
||||
|
||||
template<typename KernelFunc, typename... Args>
|
||||
void launch_kernel(const KernelLaunchConfig &cfg, KernelFunc kernel, Args&&... args) {
|
||||
if (cfg.dynamic_smem > 0) {
|
||||
KU_CUDA_CHECK(cudaFuncSetAttribute(
|
||||
reinterpret_cast<const void*>(kernel),
|
||||
cudaFuncAttributeMaxDynamicSharedMemorySize,
|
||||
static_cast<int>(cfg.dynamic_smem)));
|
||||
}
|
||||
|
||||
const bool is_cluster = !(cfg.cluster.x == 1 && cfg.cluster.y == 1 && cfg.cluster.z == 1);
|
||||
const bool need_kernel_ex = is_cluster || cfg.use_pdl;
|
||||
|
||||
if (is_cluster && cfg.cooperative) {
|
||||
KU_ASSERT(false, "cluster and cooperative launch are mutually exclusive");
|
||||
}
|
||||
|
||||
if (cfg.cooperative) {
|
||||
void* kernel_args[sizeof...(Args) > 0 ? sizeof...(Args) : 1] = {};
|
||||
if constexpr (sizeof...(Args) > 0) {
|
||||
size_t i = 0;
|
||||
((kernel_args[i++] = detail::kernel_arg_ptr(std::forward<Args>(args))), ...);
|
||||
}
|
||||
KU_CUDA_CHECK(cudaLaunchCooperativeKernel(
|
||||
kernel, cfg.grid, cfg.block,
|
||||
sizeof...(Args) > 0 ? kernel_args : nullptr,
|
||||
static_cast<unsigned int>(cfg.dynamic_smem), cfg.stream));
|
||||
} else if (need_kernel_ex) {
|
||||
if (is_cluster) {
|
||||
KU_ASSERT(cfg.grid.x % cfg.cluster.x == 0 &&
|
||||
cfg.grid.y % cfg.cluster.y == 0 &&
|
||||
cfg.grid.z % cfg.cluster.z == 0);
|
||||
if (cfg.cluster.x * cfg.cluster.y * cfg.cluster.z > 8) {
|
||||
KU_CUDA_CHECK(cudaFuncSetAttribute(
|
||||
reinterpret_cast<const void*>(kernel),
|
||||
cudaFuncAttributeNonPortableClusterSizeAllowed, 1));
|
||||
}
|
||||
}
|
||||
|
||||
const unsigned int num_attrs = is_cluster ? 2 : 1;
|
||||
cudaLaunchAttribute attrs[2];
|
||||
if (is_cluster) {
|
||||
attrs[0].id = cudaLaunchAttributeClusterDimension;
|
||||
attrs[0].val.clusterDim = {cfg.cluster.x, cfg.cluster.y, cfg.cluster.z};
|
||||
}
|
||||
attrs[num_attrs - 1].id = cudaLaunchAttributeProgrammaticStreamSerialization;
|
||||
attrs[num_attrs - 1].val.programmaticStreamSerializationAllowed = cfg.use_pdl ? 1 : 0;
|
||||
|
||||
cudaLaunchConfig_t config = {
|
||||
{cfg.grid.x, cfg.grid.y, cfg.grid.z},
|
||||
{cfg.block.x, cfg.block.y, cfg.block.z},
|
||||
cfg.dynamic_smem,
|
||||
cfg.stream,
|
||||
attrs,
|
||||
num_attrs
|
||||
};
|
||||
KU_CUDA_CHECK(cudaLaunchKernelEx(
|
||||
&config, kernel, std::forward<Args>(args)...));
|
||||
} else {
|
||||
kernel<<<cfg.grid, cfg.block, cfg.dynamic_smem, cfg.stream>>>(
|
||||
std::forward<Args>(args)...);
|
||||
}
|
||||
|
||||
KU_CHECK_KERNEL_LAUNCH();
|
||||
}
|
||||
|
||||
#endif // KERUTILS_IS_BUILD_ON_CUDA
|
||||
|
||||
} // namespace kerutils
|
||||
Vendored
+9
@@ -0,0 +1,9 @@
|
||||
#pragma once
|
||||
|
||||
#include "kerutils/common/common.h"
|
||||
|
||||
#include "host/host.h"
|
||||
|
||||
#ifdef KERUTILS_IS_BUILD_ON_CUDA
|
||||
#include "device/device.cuh"
|
||||
#endif
|
||||
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2025 DeepSeek
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
@@ -0,0 +1,90 @@
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <cuda_bf16.h>
|
||||
|
||||
namespace topk_select_common {
|
||||
|
||||
__device__ __forceinline__
|
||||
uint32_t bf16x2_to_u32(const nv_bfloat162 &v) {
|
||||
static_assert(sizeof(nv_bfloat162) == sizeof(uint32_t));
|
||||
return *reinterpret_cast<const uint32_t *>(&v);
|
||||
}
|
||||
|
||||
__device__ __forceinline__
|
||||
nv_bfloat162 u32_to_bf16x2(uint32_t u) {
|
||||
static_assert(sizeof(nv_bfloat162) == sizeof(uint32_t));
|
||||
nv_bfloat162 v;
|
||||
*reinterpret_cast<uint32_t *>(&v) = u;
|
||||
return v;
|
||||
}
|
||||
|
||||
// Distort and un-distort: Map IEEE 754 floating-point total order to unsigned integer order.
|
||||
// IEEE 754 bit representation (sign-magnitude with biased exponent and mantissa) does not
|
||||
// preserve value order when interpreted as unsigned integer, and distort() fixes this:
|
||||
// - For positive values (sign bit = 0): flip only the sign bit, mapping all positives to [0x80..., 0xFF...]
|
||||
// - For negative values (sign bit = 1): flip all bits (~), inverting the magnitude order so that
|
||||
// more negative (smaller value) => smaller uint
|
||||
// After distortion, all negatives (0x00... ~ 0x7F...) < all positives (0x80... ~ 0xFF...) in
|
||||
// unsigned integer comparison, which matches the floating-point total order.
|
||||
//
|
||||
// un_distort() is the exact inverse: flips the sign bit back for positives, flips all bits back for negatives.
|
||||
template<typename UIntValueT>
|
||||
__device__ __forceinline__
|
||||
UIntValueT distort(const UIntValueT &x) {
|
||||
static_assert(sizeof(UIntValueT) == 2 || sizeof(UIntValueT) == 4);
|
||||
if constexpr (sizeof(UIntValueT) == 2) {
|
||||
UIntValueT mask = (x&0x8000) ? 0xFFFF : 0x8000;
|
||||
return x ^ mask;
|
||||
} else {
|
||||
// mask = (x >> 31) | 0x80000000 is 0x80000000 for positives and 0xFFFFFFFF for negatives
|
||||
uint32_t t, d;
|
||||
asm ("shr.s32 %0, %2, 31;\n"
|
||||
"lop3.b32 %1, %2, %0, 0x80000000, 0x1e;\n"
|
||||
: "=r"(t), "=r"(d)
|
||||
: "r"((uint32_t)x));
|
||||
return (UIntValueT)d;
|
||||
}
|
||||
}
|
||||
|
||||
// A SIMD-like version of `distort`, which uses more efficient instructions for calculation
|
||||
template<typename UIntValueT>
|
||||
__device__ __forceinline__
|
||||
void distort_x2(UIntValueT result[2], const UIntValueT input[2]) {
|
||||
static_assert(sizeof(UIntValueT) == 2 || sizeof(UIntValueT) == 4);
|
||||
if constexpr (sizeof(UIntValueT) == 2) {
|
||||
asm volatile (
|
||||
"{"
|
||||
".reg .u32 mask;"
|
||||
"prmt.b32 mask, %1, 0, 0xbb99;" // mask[0:16] will be 0xFF if cur_value_packed[0]'s MSB is 1, 0x00 otherwise
|
||||
"or.b32 mask, mask, 0x80008000;"
|
||||
"xor.b32 %0, mask, %1;"
|
||||
"}"
|
||||
: "=r"(*(uint32_t*)(result))
|
||||
: "r"(*(uint32_t*)(input))
|
||||
);
|
||||
} else {
|
||||
result[0] = distort(input[0]);
|
||||
result[1] = distort(input[1]);
|
||||
}
|
||||
}
|
||||
|
||||
template<typename UIntValueT>
|
||||
__device__ __forceinline__
|
||||
UIntValueT un_distort(const UIntValueT &x) {
|
||||
static_assert(sizeof(UIntValueT) == 2 || sizeof(UIntValueT) == 4);
|
||||
if constexpr (sizeof(UIntValueT) == 2) {
|
||||
UIntValueT mask = (x&0x8000) ? 0x8000 : 0xFFFF;
|
||||
return x ^ mask;
|
||||
} else {
|
||||
// mask = ~(x >> 31) | 0x80000000 is 0xFFFFFFFF for positives and 0x80000000 for negatives
|
||||
uint32_t t, d;
|
||||
asm ("shr.s32 %0, %2, 31;\n"
|
||||
"lop3.b32 %1, %2, %0, 0x80000000, 0x4b;\n"
|
||||
: "=r"(t), "=r"(d)
|
||||
: "r"((uint32_t)x));
|
||||
return (UIntValueT)d;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace topk_select_common
|
||||
+1517
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,27 @@
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
|
||||
template<
|
||||
typename ValueT_, typename OutIdxT_,
|
||||
bool sorted_value_, bool sorted_index_, bool return_value_,
|
||||
uint32_t max_topk_,
|
||||
uint32_t num_threads_, uint32_t target_occupancy_,
|
||||
uint32_t elements_per_round_, uint32_t reconstruct_threshold_, uint32_t tma_buffer_depth_, uint32_t elements_per_segment_ = 512,
|
||||
uint32_t cluster_size_ = 1
|
||||
>
|
||||
struct TopkSelectConfig {
|
||||
using ValueT = ValueT_;
|
||||
using OutIdxT = OutIdxT_;
|
||||
static constexpr bool sorted_value = sorted_value_;
|
||||
static constexpr bool sorted_index = sorted_index_;
|
||||
static constexpr bool return_value = return_value_;
|
||||
static constexpr uint32_t max_topk = max_topk_;
|
||||
static constexpr uint32_t num_threads = num_threads_;
|
||||
static constexpr uint32_t target_occupancy = target_occupancy_; // Number of CTAs per SM
|
||||
static constexpr uint32_t elements_per_round = elements_per_round_;
|
||||
static constexpr uint32_t reconstruct_threshold = reconstruct_threshold_;
|
||||
static constexpr uint32_t tma_buffer_depth = tma_buffer_depth_;
|
||||
static constexpr uint32_t elements_per_segment = elements_per_segment_;
|
||||
static constexpr uint32_t cluster_size = cluster_size_; // (1 = not a cluster variant)
|
||||
};
|
||||
@@ -0,0 +1,54 @@
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
|
||||
template<typename T>
|
||||
__device__ __forceinline__ T warp_level_inclusive_prefix_sum(T x, uint32_t lane_idx) {
|
||||
static_assert(sizeof(T) == 4);
|
||||
#pragma unroll
|
||||
for (uint32_t i = 1; i <= 16; i <<= 1) {
|
||||
asm volatile (
|
||||
"{\n"
|
||||
".reg .pred p;\n"
|
||||
".reg .b32 t;\n"
|
||||
"shfl.sync.up.b32 t|p, %0, %1, 0, 0xffffffff;\n"
|
||||
"@p add.u32 %0, %0, t;\n"
|
||||
"}\n"
|
||||
: "+r"(x)
|
||||
: "r"(i)
|
||||
);
|
||||
}
|
||||
return x;
|
||||
}
|
||||
|
||||
template<typename T>
|
||||
__device__ __forceinline__ T warp_level_exclusive_prefix_sum(T x, uint32_t lane_idx) {
|
||||
T inclusive_prefix_sum = warp_level_inclusive_prefix_sum(x, lane_idx);
|
||||
return inclusive_prefix_sum - x;
|
||||
}
|
||||
|
||||
template<typename T>
|
||||
__device__ __forceinline__ T warp_level_inclusive_suffix_sum(T x, uint32_t lane_idx) {
|
||||
static_assert(sizeof(T) == 4);
|
||||
#pragma unroll
|
||||
for (uint32_t i = 1; i <= 16; i <<= 1) {
|
||||
asm volatile (
|
||||
"{\n"
|
||||
".reg .pred p;\n"
|
||||
".reg .b32 t;\n"
|
||||
"shfl.sync.down.b32 t|p, %0, %1, 31, 0xffffffff;\n"
|
||||
"@p add.u32 %0, %0, t;\n"
|
||||
"}\n"
|
||||
: "+r"(x)
|
||||
: "r"(i)
|
||||
);
|
||||
}
|
||||
return x;
|
||||
}
|
||||
|
||||
|
||||
template<typename T>
|
||||
__device__ __forceinline__ T warp_level_exclusive_suffix_sum(T x, uint32_t lane_idx) {
|
||||
T inclusive_suffix_sum = warp_level_inclusive_suffix_sum(x, lane_idx);
|
||||
return inclusive_suffix_sum - x;
|
||||
}
|
||||
+147
@@ -0,0 +1,147 @@
|
||||
/*
|
||||
TopK select kernel bf16 common variant.
|
||||
|
||||
Usage caveats:
|
||||
- vocab_size must be < 2^23.
|
||||
- Do NOT assume that the kernel will select a prefix index on ties.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <cutlass/kernel_launch.h>
|
||||
#include <cute/arch/copy_sm90_tma.hpp>
|
||||
#include <kerutils/kerutils.cuh>
|
||||
|
||||
#include "structs.h"
|
||||
#include "cuda_kernels/utils.cuh"
|
||||
#include "cuda_kernels/common_parts.cuh"
|
||||
|
||||
namespace topk_select_bf16_normal {
|
||||
|
||||
template<typename Config>
|
||||
class TopkSelectKernelBF16Normal : public topk_select_common::TopkSelectKernelBF16Base<Config> {
|
||||
using BF16Base = topk_select_common::TopkSelectKernelBF16Base<Config>;
|
||||
public:
|
||||
using ValueT = typename BF16Base::ValueT;
|
||||
using OutIdxT = typename BF16Base::OutIdxT;
|
||||
using TmaParams = typename BF16Base::TmaParams;
|
||||
using SharedMemoryPlanBase = typename BF16Base::SharedMemoryPlanBase;
|
||||
using EpilogueT = typename BF16Base::EpilogueT;
|
||||
using BF16Base::NUM_THREADS;
|
||||
using BF16Base::MAX_TOPK;
|
||||
using BF16Base::NUM_ELEMS_PER_SEG;
|
||||
using BF16Base::NUM_ELEMS_PER_ROUND;
|
||||
using BF16Base::NUM_SEGS_IN_INIT_WINDOW;
|
||||
using BF16Base::NUM_INIT_ROUNDS_MAX;
|
||||
using BF16Base::NUM_TAIL_ELEMS;
|
||||
using BF16Base::NUM_TAIL_SEGS;
|
||||
using BF16Base::NUM_SEGS_PER_ROUND;
|
||||
|
||||
static_assert(Config::target_occupancy >= 1 && Config::target_occupancy <= 8);
|
||||
static_assert(MAX_TOPK == 512 || MAX_TOPK == 1024 || MAX_TOPK == 4096);
|
||||
static constexpr bool HAS_PARTIAL_ROUNDS = NUM_SEGS_PER_ROUND > NUM_TAIL_SEGS;
|
||||
struct SharedMemoryPlan : SharedMemoryPlanBase {};
|
||||
|
||||
static __device__ __forceinline__
|
||||
void topk_select_kernel_devfunc(const TopkSelectArgs &args, const TmaParams &tma_params) {
|
||||
uint32_t batch_idx = blockIdx.x;
|
||||
uint32_t end_vocab_idx = args.end_ptr == nullptr ? args.vocab_size : __ldg(args.end_ptr + batch_idx);
|
||||
|
||||
extern __shared__ CUTE_ALIGNAS(1024) char wksp_buf[];
|
||||
SharedMemoryPlan &smem = *reinterpret_cast<SharedMemoryPlan*>(wksp_buf);
|
||||
|
||||
uint32_t warp_idx = cutlass::canonical_warp_idx_sync();
|
||||
uint32_t lane_idx = threadIdx.x % 32;
|
||||
|
||||
if (end_vocab_idx <= args.topk) {
|
||||
EpilogueT::template topk_select_epilogue<true>(
|
||||
(ValueT*)smem.surviving_topk_pairs[0],
|
||||
(uint32_t*)(smem.surviving_topk_pairs[1]),
|
||||
args,
|
||||
batch_idx, end_vocab_idx, warp_idx,
|
||||
*reinterpret_cast<typename EpilogueT::BlockRadixSortTempStorageT*>(smem.incoming_topk_pairs)
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
uint32_t survivor_buf_idx = 0; // Current candidate buffer (A/B, swapped by reconstruct)
|
||||
bool have_nan = false;
|
||||
|
||||
BF16Base::init_shared_memory(smem, warp_idx);
|
||||
|
||||
uint32_t num_input_segs = ku::ceil_div(end_vocab_idx, (uint32_t)NUM_ELEMS_PER_SEG);
|
||||
uint32_t num_perm_segs = num_input_segs <= NUM_SEGS_IN_INIT_WINDOW ? 0u : (num_input_segs - 1) / NUM_TAIL_SEGS * NUM_TAIL_SEGS; // Skip permutation if the total number of segs is small enough
|
||||
uint32_t num_perm_elems = num_perm_segs * NUM_ELEMS_PER_SEG;
|
||||
uint32_t num_tail_elems_padded = num_perm_segs != 0 ? (uint32_t)NUM_TAIL_ELEMS : num_input_segs * NUM_ELEMS_PER_SEG;
|
||||
uint32_t num_rounds = ku::ceil_div(num_tail_elems_padded + num_perm_elems, (uint32_t)NUM_ELEMS_PER_ROUND);
|
||||
uint32_t num_init_rounds = min(num_rounds, (uint32_t)NUM_INIT_ROUNDS_MAX);
|
||||
|
||||
BF16Base::template scan_segs<false>(
|
||||
tma_params, smem,
|
||||
batch_idx, end_vocab_idx, args.topk,
|
||||
warp_idx, lane_idx,
|
||||
num_perm_segs,
|
||||
0,
|
||||
num_perm_segs,
|
||||
num_tail_elems_padded,
|
||||
end_vocab_idx - num_perm_elems,
|
||||
survivor_buf_idx,
|
||||
have_nan,
|
||||
[&](uint32_t m) {
|
||||
if constexpr (HAS_PARTIAL_ROUNDS) {
|
||||
return (num_init_rounds + m) * NUM_SEGS_PER_ROUND + warp_idx < NUM_TAIL_SEGS + num_perm_segs;
|
||||
} else {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
// `__syncthreads_or` is also the barrier that makes the last `reconstruct`'s writes visible before
|
||||
// `stage_output_and_epilogue` reads the survivor buffer.
|
||||
have_nan = __syncthreads_or(have_nan) != 0;
|
||||
if (have_nan) {
|
||||
BF16Base::take_action_when_have_nan(args, batch_idx);
|
||||
return;
|
||||
}
|
||||
|
||||
BF16Base::template stage_output_and_epilogue<Config::sorted_index>(
|
||||
smem, *reinterpret_cast<typename EpilogueT::BlockRadixSortTempStorageT*>(smem.incoming_topk_pairs), args,
|
||||
batch_idx, end_vocab_idx, survivor_buf_idx, warp_idx, lane_idx);
|
||||
}
|
||||
};
|
||||
|
||||
template<typename Kernel>
|
||||
__launch_bounds__(Kernel::NUM_THREADS, Kernel::TARGET_OCCUPANCY, 1)
|
||||
__global__ void topk_kernel(__grid_constant__ const TopkSelectArgs args, __grid_constant__ const typename Kernel::TmaParams tma_params) {
|
||||
Kernel::topk_select_kernel_devfunc(args, tma_params);
|
||||
}
|
||||
|
||||
template<typename Config>
|
||||
void run_topk_select_kernel(const TopkSelectArgs &args) {
|
||||
KU_ASSERT(args.sorted_value == Config::sorted_value, "Dispatch failure");
|
||||
KU_ASSERT(args.sorted_index == Config::sorted_index, "Dispatch failure");
|
||||
KU_ASSERT(args.return_value == Config::return_value, "Dispatch failure");
|
||||
static_assert(cute::is_same_v<typename Config::ValueT, nv_bfloat16>);
|
||||
KU_ASSERT(args.vocab_size < MAX_VOCAB_SIZE, "`vocab_size` is too big");
|
||||
|
||||
using Kernel = TopkSelectKernelBF16Normal<Config>;
|
||||
KU_ASSERT(args.topk <= Kernel::MAX_TOPK, "topk is too large. Maximum allowed: %d\n", Kernel::MAX_TOPK);
|
||||
static_assert(INPUT_STRIDE_ALIGNMENT_REQUIREMENT % 16 == 0);
|
||||
|
||||
auto kernel = topk_kernel<Kernel>;
|
||||
constexpr size_t smem_size = sizeof(typename Kernel::SharedMemoryPlan);
|
||||
KU_ASSERT(smem_size * Kernel::TARGET_OCCUPANCY <= args.shared_memory_size_per_sm);
|
||||
KU_CUDA_CHECK(cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, smem_size));
|
||||
|
||||
KU_ASSERT(args.stride_input_batch % 8 == 0, "stride_input_batch must be 16B-aligned");
|
||||
typename Kernel::TmaParams tma_params = {Kernel::make_topk_tensor_map(args)};
|
||||
|
||||
ku::launch_kernel(ku::KernelLaunchConfig {
|
||||
dim3(args.batch_size),
|
||||
dim3(Kernel::NUM_THREADS),
|
||||
smem_size,
|
||||
args.stream
|
||||
}, kernel, args, tma_params);
|
||||
}
|
||||
|
||||
} // topk_select_bf16_normal
|
||||
+305
@@ -0,0 +1,305 @@
|
||||
/*
|
||||
TopK select kernel bf16 cluster variant. Use multiple CTAs to calculate one sequence to avoid wave quantization
|
||||
|
||||
Algorithm:
|
||||
- Each CTA computes its local top-k pairs
|
||||
- Each CTA sends its local top-k pairs to CTA0
|
||||
- CTA0 selects top-k from those pairs
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <cutlass/kernel_launch.h>
|
||||
#include <cute/arch/cluster_sm90.hpp>
|
||||
#include <cute/arch/copy_sm90_tma.hpp>
|
||||
#include <kerutils/kerutils.cuh>
|
||||
|
||||
#include "structs.h"
|
||||
#include "cuda_kernels/utils.cuh"
|
||||
#include "cuda_kernels/common_parts.cuh"
|
||||
|
||||
namespace topk_select_bf16_cluster {
|
||||
|
||||
CUTE_DEVICE
|
||||
static void st_async_32b(uint32_t dst_addr, const uint32_t& data, transac_bar_t &mbar) {
|
||||
uint32_t mbar_addr = cute::cast_smem_ptr_to_uint(&mbar);
|
||||
asm volatile (
|
||||
"st.async.weak.shared::cluster.mbarrier::complete_tx::bytes.s32 [%0], {%1}, [%2]; \n"
|
||||
:
|
||||
: "r"(dst_addr), "r"(data), "r"(mbar_addr)
|
||||
);
|
||||
}
|
||||
|
||||
template<typename Config>
|
||||
class TopkSelectKernelBF16Cluster : public topk_select_common::TopkSelectKernelBF16Base<Config> {
|
||||
using BF16Base = topk_select_common::TopkSelectKernelBF16Base<Config>;
|
||||
public:
|
||||
using ValueT = typename BF16Base::ValueT;
|
||||
using OutIdxT = typename BF16Base::OutIdxT;
|
||||
using TmaParams = typename BF16Base::TmaParams;
|
||||
using SharedMemoryPlanBase = typename BF16Base::SharedMemoryPlanBase;
|
||||
using EpilogueT = typename BF16Base::EpilogueT;
|
||||
using BF16Base::NUM_THREADS;
|
||||
using BF16Base::MAX_TOPK;
|
||||
using BF16Base::NUM_EXTRA_SLOTS;
|
||||
using BF16Base::NUM_ELEMS_PER_SEG;
|
||||
using BF16Base::NUM_ELEMS_PER_ROUND;
|
||||
using BF16Base::NUM_SEGS_IN_INIT_WINDOW;
|
||||
using BF16Base::NUM_INIT_ROUNDS_MAX;
|
||||
using BF16Base::NUM_TAIL_ELEMS;
|
||||
using BF16Base::NUM_TAIL_SEGS;
|
||||
using BF16Base::NUM_SEGS_PER_ROUND;
|
||||
|
||||
static_assert(Config::target_occupancy == 1);
|
||||
static_assert(MAX_TOPK == 512 || MAX_TOPK == 1024);
|
||||
static_assert(Config::cluster_size == 8 || Config::cluster_size == 16);
|
||||
static_assert(NUM_ELEMS_PER_ROUND * sizeof(ValueT) % 1024 == 0);
|
||||
static_assert((uint64_t)(MAX_VOCAB_SIZE / NUM_ELEMS_PER_SEG) * (MAX_VOCAB_SIZE / NUM_ELEMS_PER_SEG)
|
||||
+ BF16Base::PERM_ADD_BASE <= 0xFFFFFFFFull);
|
||||
static constexpr uint32_t CLUSTER_SIZE = Config::cluster_size;
|
||||
static constexpr uint32_t NUM_GATHER_PAIRS = Config::cluster_size * MAX_TOPK;
|
||||
static constexpr uint32_t NUM_GATHER_UNITS = NUM_GATHER_PAIRS / 2;
|
||||
static constexpr uint32_t NUM_UINT32_GATHER_PER_THREAD = ((NUM_GATHER_UNITS + NUM_THREADS - 1) / NUM_THREADS) | 1u;
|
||||
struct SharedMemoryPlanBF16Cluster : SharedMemoryPlanBase {
|
||||
kerutils::transac_bar_t gather_val_bar;
|
||||
kerutils::transac_bar_t gather_bar;
|
||||
CUTE_ALIGNAS(16) uint32_t gathered_num_survivors[Config::cluster_size];
|
||||
};
|
||||
static_assert(NUM_GATHER_UNITS % NUM_THREADS == 0);
|
||||
static_assert(NUM_GATHER_PAIRS <= 0xFFFF);
|
||||
static_assert(2 * NUM_UINT32_GATHER_PER_THREAD <= 256);
|
||||
|
||||
static __device__ __forceinline__ void topk_select_kernel_devfunc(const TopkSelectArgs &args, const TmaParams &tma_params) {
|
||||
uint32_t rank_in_cluster = blockIdx.x;
|
||||
uint32_t batch_idx = blockIdx.y;
|
||||
uint32_t end_vocab_idx = args.end_ptr == nullptr ? args.vocab_size : __ldg(args.end_ptr + batch_idx);
|
||||
|
||||
extern __shared__ CUTE_ALIGNAS(1024) char wksp_buf[];
|
||||
SharedMemoryPlanBF16Cluster &smem = *reinterpret_cast<SharedMemoryPlanBF16Cluster*>(wksp_buf);
|
||||
|
||||
uint32_t warp_idx = cutlass::canonical_warp_idx_sync();
|
||||
uint32_t lane_idx = threadIdx.x % 32;
|
||||
|
||||
if (end_vocab_idx <= args.topk) {
|
||||
if (rank_in_cluster != 0) {
|
||||
return;
|
||||
}
|
||||
EpilogueT::template topk_select_epilogue<true>(
|
||||
(ValueT*)smem.surviving_topk_pairs[0],
|
||||
(uint32_t*)(smem.surviving_topk_pairs[1]),
|
||||
args,
|
||||
batch_idx, end_vocab_idx, warp_idx,
|
||||
*reinterpret_cast<typename EpilogueT::BlockRadixSortTempStorageT*>(smem.incoming_topk_pairs)
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
uint32_t survivor_buf_idx = 0; // current candidate buffer (A/B, swapped by reconstruct)
|
||||
|
||||
BF16Base::init_shared_memory(smem, warp_idx, [&]{
|
||||
smem.gather_val_bar.init(1);
|
||||
smem.gather_bar.init(1);
|
||||
});
|
||||
|
||||
// Still, divide the whole input into the "perm" part and the "tail" part
|
||||
// The "tail" part contains the last 1 ~ NUM_TAIL_ELEMS elements and the "perm" part has the other elements.
|
||||
uint32_t num_input_segs = ku::ceil_div(end_vocab_idx, (uint32_t)NUM_ELEMS_PER_SEG);
|
||||
uint32_t num_perm_segs = num_input_segs <= NUM_SEGS_IN_INIT_WINDOW ? 0u : (num_input_segs - 1) / NUM_TAIL_SEGS * NUM_TAIL_SEGS;
|
||||
uint32_t num_perm_elems = num_perm_segs * NUM_ELEMS_PER_SEG;
|
||||
uint32_t num_tail_elems_padded = num_perm_segs != 0 ? (uint32_t)NUM_TAIL_ELEMS : num_input_segs * NUM_ELEMS_PER_SEG;
|
||||
|
||||
// The "tail" part is handled by CTA0 in the cluster
|
||||
// "local" means the current CTA (in the cluster)
|
||||
uint32_t num_local_tail_elems_padded = rank_in_cluster == 0 ? num_tail_elems_padded : 0u;
|
||||
uint32_t num_local_tail_segs = num_local_tail_elems_padded / NUM_ELEMS_PER_SEG;
|
||||
uint32_t num_local_tail_elems = rank_in_cluster == 0 ? end_vocab_idx - num_perm_elems : 0u;
|
||||
|
||||
// rank r takes the r-th contiguous interval of the row's visit order, with equal visit elements per rank
|
||||
uint32_t num_total_elems_padded = num_tail_elems_padded + num_perm_elems;
|
||||
// Rank r's slice of the padded row starts at r * num_total_elems_padded / cluster_size; the tail is
|
||||
// rank 0's alone, so clamp that boundary into the perm part (element index within the perm part).
|
||||
auto perm_boundary = [&](uint32_t rank) {
|
||||
static_assert((uint64_t)Config::cluster_size * (MAX_VOCAB_SIZE + NUM_TAIL_ELEMS) <= 0xFFFFFFFFull);
|
||||
uint32_t boundary = rank * num_total_elems_padded / Config::cluster_size;
|
||||
return boundary > num_tail_elems_padded ? boundary - num_tail_elems_padded : 0u;
|
||||
};
|
||||
uint32_t local_start_seg_idx = perm_boundary(rank_in_cluster) / NUM_ELEMS_PER_SEG;
|
||||
uint32_t local_end_seg_idx = perm_boundary(rank_in_cluster + 1) / NUM_ELEMS_PER_SEG;
|
||||
uint32_t num_local_perm_segs = local_end_seg_idx - local_start_seg_idx;
|
||||
uint32_t num_local_elems = num_local_tail_elems_padded + num_local_perm_segs * NUM_ELEMS_PER_SEG;
|
||||
|
||||
uint32_t num_local_rounds = ku::ceil_div(num_local_elems, (uint32_t)NUM_ELEMS_PER_ROUND);
|
||||
uint32_t num_local_init_rounds = min(num_local_rounds, (uint32_t)NUM_INIT_ROUNDS_MAX);
|
||||
|
||||
uint32_t num_local_segs = num_local_tail_segs + num_local_perm_segs;
|
||||
uint32_t used_segs = num_local_init_rounds * NUM_SEGS_PER_ROUND;
|
||||
uint32_t rem_segs = num_local_segs > used_segs ? num_local_segs - used_segs : 0u;
|
||||
uint32_t warp_rem_segs = warp_idx < rem_segs ? rem_segs - warp_idx : 0u;
|
||||
uint32_t warp_round_limit = ku::ceil_div(warp_rem_segs, (uint32_t)NUM_SEGS_PER_ROUND);
|
||||
|
||||
// Each CTA gets its local top-k elements
|
||||
bool nan_seen = false;
|
||||
BF16Base::template scan_segs<true>(
|
||||
tma_params, smem,
|
||||
batch_idx, end_vocab_idx, args.topk, warp_idx, lane_idx,
|
||||
num_perm_segs, local_start_seg_idx, num_local_perm_segs, num_local_tail_elems_padded, num_local_tail_elems,
|
||||
survivor_buf_idx,
|
||||
nan_seen,
|
||||
[&](uint32_t m) { return m < warp_round_limit; }
|
||||
);
|
||||
|
||||
// CTA-wide OR; this is also the barrier that makes the scan's writes to the survivor buffer visible to the gather below
|
||||
nan_seen = __syncthreads_or(nan_seen) != 0;
|
||||
|
||||
// The cluster barrier must not complete before every CTA has finished its own scan: the gather
|
||||
// below writes into CTA0's shared memory, so a CTA that finished early would otherwise clobber
|
||||
// CTA0's `tma_load_buf` / `incoming_topk_pairs` while CTA0 is still scanning them.
|
||||
ku::barrier_cluster_arrive_relaxed(); // Paired with the `barrier_cluster_wait_acquire` below
|
||||
ku::barrier_cluster_wait_acquire();
|
||||
|
||||
static_assert(NUM_GATHER_UNITS * sizeof(uint32_t) <= sizeof(smem.incoming_topk_pairs));
|
||||
void* val_dst = (void*)(int64_t)cute::set_block_rank(
|
||||
cute::cast_smem_ptr_to_uint((uint32_t*)smem.incoming_topk_pairs + rank_in_cluster * (MAX_TOPK / 2)), 0);
|
||||
|
||||
static_assert(NUM_GATHER_PAIRS * sizeof(uint64_t) <= sizeof(SharedMemoryPlanBase::tma_load_buf));
|
||||
void* dst = (void*)(int64_t)cute::set_block_rank(
|
||||
cute::cast_smem_ptr_to_uint(reinterpret_cast<uint64_t*>(smem.tma_load_buf) + rank_in_cluster * MAX_TOPK), 0);
|
||||
|
||||
static_assert(MAX_TOPK*sizeof(ValueT) % 16 == 0);
|
||||
constexpr uint32_t NUM_VAL_CHUNKS = MAX_TOPK * sizeof(ValueT) / 16; // 16B store
|
||||
constexpr uint32_t NUM_VAL_CHUNKS_PER_THREAD = ku::ceil_div(NUM_VAL_CHUNKS, NUM_THREADS);
|
||||
|
||||
if (warp_idx == 0 && cute::elect_one_sync()) {
|
||||
if (rank_in_cluster == 0) {
|
||||
smem.gather_val_bar.arrive_and_expect_tx((NUM_VAL_CHUNKS * 16 + sizeof(uint32_t)) * CLUSTER_SIZE);
|
||||
smem.gather_bar.arrive_and_expect_tx((MAX_TOPK * (uint32_t)sizeof(uint64_t)) * CLUSTER_SIZE);
|
||||
}
|
||||
st_async_32b(
|
||||
cute::set_block_rank(cute::cast_smem_ptr_to_uint(smem.gathered_num_survivors + rank_in_cluster), 0),
|
||||
(uint32_t)nan_seen,
|
||||
smem.gather_val_bar
|
||||
);
|
||||
}
|
||||
|
||||
// Store values first... so that CTA0 can begin pivot selection earlier
|
||||
CUTE_UNROLL
|
||||
for (uint32_t i = 0; i < NUM_VAL_CHUNKS_PER_THREAD; ++i) {
|
||||
uint32_t c = i * NUM_THREADS + threadIdx.x;
|
||||
if constexpr (NUM_VAL_CHUNKS % NUM_THREADS != 0) {
|
||||
if (c >= NUM_VAL_CHUNKS) break;
|
||||
}
|
||||
uint32_t vw[4];
|
||||
CUTE_UNROLL
|
||||
for (uint32_t j = 0; j < 4; ++j) {
|
||||
uint32_t pair2[4];
|
||||
topk_select_common::ld_shared<4>(pair2, (const uint32_t*)(smem.surviving_topk_pairs[survivor_buf_idx] + 8 * c + 2 * j));
|
||||
vw[j] = __byte_perm(pair2[1], pair2[3], 0x5410);
|
||||
}
|
||||
ku::st_async((char*)val_dst + c * sizeof(uint4), make_uint4(vw[0], vw[1], vw[2], vw[3]),
|
||||
smem.gather_val_bar);
|
||||
}
|
||||
|
||||
// And then store those (value, index) pairs
|
||||
static_assert(MAX_TOPK * sizeof(uint64_t) % 16 == 0);
|
||||
constexpr uint32_t NUM_INDEX_VALUE_CHUNKS = MAX_TOPK * sizeof(uint64_t) / 16;
|
||||
static_assert(NUM_INDEX_VALUE_CHUNKS % NUM_THREADS == 0);
|
||||
CUTE_UNROLL
|
||||
for (uint32_t i = 0; i < NUM_INDEX_VALUE_CHUNKS / NUM_THREADS; ++i) {
|
||||
uint32_t s = 2 * (i * NUM_THREADS + threadIdx.x);
|
||||
ulonglong2 pp = *reinterpret_cast<const ulonglong2*>(smem.surviving_topk_pairs[survivor_buf_idx] + s);
|
||||
ku::st_async((char*)dst + s * sizeof(uint64_t), pp, smem.gather_bar);
|
||||
}
|
||||
|
||||
if (rank_in_cluster != 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
// The following code is CTA0 only
|
||||
smem.gather_val_bar.wait(0);
|
||||
BF16Base::clear_reconstruct_histograms(smem, threadIdx.x);
|
||||
__syncthreads();
|
||||
|
||||
static_assert(Config::cluster_size <= 32);
|
||||
uint32_t stored_num_survivors = lane_idx < Config::cluster_size ? smem.gathered_num_survivors[lane_idx] : 0u;
|
||||
nan_seen |= stored_num_survivors;
|
||||
|
||||
uint32_t unit_base = threadIdx.x * NUM_UINT32_GATHER_PER_THREAD;
|
||||
uint32_t num_my_units = unit_base < NUM_GATHER_UNITS ? min((uint32_t)NUM_UINT32_GATHER_PER_THREAD, NUM_GATHER_UNITS - unit_base) : 0u;
|
||||
|
||||
const uint32_t *gather_vals = (const uint32_t*)smem.incoming_topk_pairs;
|
||||
nv_bfloat162 values[NUM_UINT32_GATHER_PER_THREAD];
|
||||
CUTE_UNROLL
|
||||
for (uint32_t i = 0; i < NUM_UINT32_GATHER_PER_THREAD; i++) {
|
||||
if (i == num_my_units) break;
|
||||
values[i] = topk_select_common::u32_to_bf16x2(gather_vals[unit_base + i]);
|
||||
}
|
||||
BF16Base::histogram_radix_msb(smem.reconstruct_bucket_counter[0], values, num_my_units);
|
||||
__syncthreads();
|
||||
|
||||
auto [pivot_value_x2_bits, out_prefix, eq_quota, cnt_nan] =
|
||||
BF16Base::template compute_pivot_and_quota<true>(args.topk, values, num_my_units, warp_idx, lane_idx, smem);
|
||||
nan_seen |= cnt_nan != 0;
|
||||
|
||||
smem.gather_bar.wait(0);
|
||||
__syncthreads();
|
||||
|
||||
uint32_t out_ptr = cute::cast_smem_ptr_to_uint(smem.surviving_topk_pairs[survivor_buf_idx ^ 1]) + out_prefix * (uint32_t)sizeof(uint64_t);
|
||||
uint32_t gather_pairs_base = cute::cast_smem_ptr_to_uint(smem.tma_load_buf);
|
||||
CUTE_UNROLL
|
||||
for (uint32_t m = 0; m < NUM_UINT32_GATHER_PER_THREAD; m++) {
|
||||
if (m == num_my_units) break;
|
||||
BF16Base::template copy_selected_pairs_to_survivor<true>(out_ptr, eq_quota, values[m], pivot_value_x2_bits, gather_pairs_base + 2 * (unit_base + m) * (uint32_t)sizeof(uint64_t));
|
||||
}
|
||||
survivor_buf_idx ^= 1;
|
||||
|
||||
nan_seen = __syncthreads_or(nan_seen) != 0;
|
||||
if (nan_seen) {
|
||||
BF16Base::take_action_when_have_nan(args, batch_idx);
|
||||
return;
|
||||
}
|
||||
|
||||
BF16Base::template stage_output_and_epilogue<Config::sorted_index>(
|
||||
smem, *reinterpret_cast<typename EpilogueT::BlockRadixSortTempStorageT*>(smem.incoming_topk_pairs), args,
|
||||
batch_idx, end_vocab_idx, survivor_buf_idx, warp_idx, lane_idx);
|
||||
}
|
||||
};
|
||||
|
||||
template<typename Kernel>
|
||||
__launch_bounds__(Kernel::NUM_THREADS, Kernel::TARGET_OCCUPANCY, Kernel::CLUSTER_SIZE)
|
||||
__global__ void topk_kernel(__grid_constant__ const TopkSelectArgs args, __grid_constant__ const typename Kernel::TmaParams tma_params) {
|
||||
Kernel::topk_select_kernel_devfunc(args, tma_params);
|
||||
}
|
||||
|
||||
template<typename Config>
|
||||
void run_topk_select_kernel(const TopkSelectArgs &args) {
|
||||
KU_ASSERT(args.sorted_value == Config::sorted_value, "Dispatch failure");
|
||||
KU_ASSERT(args.sorted_index == Config::sorted_index, "Dispatch failure");
|
||||
KU_ASSERT(args.return_value == Config::return_value, "Dispatch failure");
|
||||
static_assert(cute::is_same_v<typename Config::ValueT, nv_bfloat16>);
|
||||
KU_ASSERT(args.vocab_size < MAX_VOCAB_SIZE, "`vocab_size` is too big");
|
||||
|
||||
using Kernel = TopkSelectKernelBF16Cluster<Config>;
|
||||
KU_ASSERT(args.topk <= Kernel::MAX_TOPK, "topk is too large. Maximum allowed: %d\n", Kernel::MAX_TOPK);
|
||||
static_assert(INPUT_STRIDE_ALIGNMENT_REQUIREMENT % 16 == 0);
|
||||
|
||||
auto kernel = topk_kernel<Kernel>;
|
||||
constexpr size_t smem_size = sizeof(typename Kernel::SharedMemoryPlanBF16Cluster);
|
||||
KU_ASSERT(smem_size * Kernel::TARGET_OCCUPANCY <= args.shared_memory_size_per_sm);
|
||||
KU_CUDA_CHECK(cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, smem_size));
|
||||
if constexpr (Kernel::CLUSTER_SIZE > 8) {
|
||||
KU_CUDA_CHECK(cudaFuncSetAttribute(kernel, cudaFuncAttributeNonPortableClusterSizeAllowed, 1));
|
||||
}
|
||||
|
||||
KU_ASSERT(args.stride_input_batch % 8 == 0, "stride_input_batch must be 16B-aligned");
|
||||
typename Kernel::TmaParams tma_params = {Kernel::make_topk_tensor_map(args)};
|
||||
|
||||
ku::launch_kernel(ku::KernelLaunchConfig {
|
||||
dim3(Kernel::CLUSTER_SIZE, args.batch_size),
|
||||
dim3(Kernel::NUM_THREADS),
|
||||
smem_size,
|
||||
args.stream,
|
||||
dim3(Kernel::CLUSTER_SIZE, 1, 1)
|
||||
}, kernel, args, tma_params);
|
||||
}
|
||||
|
||||
} // topk_select_bf16_cluster
|
||||
+601
@@ -0,0 +1,601 @@
|
||||
/*
|
||||
TopK select kernel fp32 variant.
|
||||
|
||||
Usage caveats:
|
||||
- vocab_size must be < 2^23.
|
||||
- Do NOT assume that the kernel will select a prefix index on ties.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <cutlass/kernel_launch.h>
|
||||
#include <cute/arch/copy_sm90_tma.hpp>
|
||||
#include <kerutils/kerutils.cuh>
|
||||
|
||||
#include "structs.h"
|
||||
#include "cuda_kernels/utils.cuh"
|
||||
#include "cuda_kernels/common_parts.cuh"
|
||||
|
||||
namespace topk_select_fp32 {
|
||||
|
||||
template<typename Config>
|
||||
class TopkSelectKernelFP32 : public topk_select_common::TopkSelectKernelBase<Config> {
|
||||
using Base = topk_select_common::TopkSelectKernelBase<Config>;
|
||||
public:
|
||||
using ValueT = typename Base::ValueT;
|
||||
using OutIdxT = typename Base::OutIdxT;
|
||||
using TmaParams = typename Base::TmaParams;
|
||||
using SharedMemoryPlanBase = typename Base::SharedMemoryPlanBase;
|
||||
using EpilogueT = typename Base::EpilogueT;
|
||||
using Base::NUM_THREADS;
|
||||
using Base::NUM_WARPS;
|
||||
using Base::MAX_TOPK;
|
||||
using Base::NUM_ELEMS_PER_128b;
|
||||
using Base::NUM_UINT32_PER_128b;
|
||||
using Base::NUM_ELEMS_PER_ROUND;
|
||||
using Base::NUM_ELEMS_PER_THREAD_PER_ROUND;
|
||||
using Base::ELEMS_PER_THREAD_PER_ROUND_MASK;
|
||||
using Base::NUM_128b_PER_THREAD_PER_ROUND;
|
||||
using Base::NUM_ELEMS_PER_SEG;
|
||||
using Base::NUM_TAIL_ELEMS;
|
||||
using Base::NUM_TAIL_SEGS;
|
||||
using Base::NUM_ELEMS_IN_INIT_WINDOW;
|
||||
using Base::NUM_SEGS_IN_INIT_WINDOW;
|
||||
using Base::NUM_PERM_SEGS_IN_INIT_WINDOW;
|
||||
using Base::NUM_INIT_ROUNDS_MAX;
|
||||
using Base::NUM_UINT32_IN_INIT_WINDOW_PER_THREAD;
|
||||
using Base::NUM_128b_INIT_PER_THREAD;
|
||||
using Base::NUM_SEGS_PER_ROUND;
|
||||
using Base::NUM_SEGS_PER_ISSUE_WARP;
|
||||
using Base::NUM_ISSUE_WARPS;
|
||||
using Base::NUM_TMA_LOAD_BUFS;
|
||||
using Base::TMA_PREFETCH_DEPTH;
|
||||
using Base::RECONSTRUCT_THRESHOLD;
|
||||
using Base::PERM_ADD_BASE;
|
||||
using Base::PERM_MUL_PRIME;
|
||||
using Base::PLACEHOLDER_PAIR;
|
||||
using Base::NUM_RECONSTRUCT_BUCKETS;
|
||||
|
||||
static constexpr uint32_t PLACEHOLDER = 0xff800000; // -INF
|
||||
static constexpr uint32_t NEG_INF_BITS = 0xFF800000;
|
||||
static_assert(MAX_TOPK == 512 || MAX_TOPK == 1024 || MAX_TOPK == 4096);
|
||||
static constexpr bool HAS_PARTIAL_ROUNDS = NUM_SEGS_PER_ROUND > NUM_TAIL_SEGS;
|
||||
|
||||
struct SharedMemoryPlanFP32 : SharedMemoryPlanBase {};
|
||||
|
||||
static __device__ __forceinline__
|
||||
void append_pair_at(uint32_t &dst_ptr, uint32_t index, uint32_t val_bits) {
|
||||
*(uint64_t*)__cvta_shared_to_generic(dst_ptr) = (uint64_t)val_bits << 32 | index;
|
||||
dst_ptr = __float_as_uint(__uint_as_float(dst_ptr) + __uint_as_float(8u));
|
||||
}
|
||||
|
||||
static __device__ __forceinline__ void topk_select_kernel_devfunc(const TopkSelectArgs &args, const TmaParams &tma_params) {
|
||||
uint32_t batch_idx = blockIdx.x;
|
||||
uint32_t end_vocab_idx = args.end_ptr == nullptr ? args.vocab_size : __ldg(args.end_ptr + batch_idx);
|
||||
|
||||
extern __shared__ CUTE_ALIGNAS(1024) char wksp_buf[];
|
||||
SharedMemoryPlanFP32 &smem = *reinterpret_cast<SharedMemoryPlanFP32*>(wksp_buf);
|
||||
|
||||
uint32_t warp_idx = cutlass::canonical_warp_idx_sync();
|
||||
uint32_t lane_idx = threadIdx.x % 32;
|
||||
|
||||
if (end_vocab_idx <= args.topk) {
|
||||
EpilogueT::template topk_select_epilogue<true>(
|
||||
(ValueT*)smem.surviving_topk_pairs[0],
|
||||
(uint32_t*)(smem.surviving_topk_pairs[1]),
|
||||
args,
|
||||
batch_idx, end_vocab_idx, warp_idx,
|
||||
*reinterpret_cast<typename EpilogueT::BlockRadixSortTempStorageT*>(smem.incoming_topk_pairs)
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
uint32_t threshold_bits = NEG_INF_BITS;
|
||||
uint32_t num_incomers = 0;
|
||||
uint32_t survivor_buf_idx = 0; // current candidate buffer (A/B, swapped by reconstruct)
|
||||
bool have_nan = false;
|
||||
|
||||
Base::init_shared_memory(smem, warp_idx);
|
||||
|
||||
uint32_t num_input_segs = ku::ceil_div(end_vocab_idx, (uint32_t)NUM_ELEMS_PER_SEG);
|
||||
uint32_t num_perm_segs = num_input_segs <= NUM_SEGS_IN_INIT_WINDOW ? 0u : (num_input_segs - 1) / NUM_TAIL_SEGS * NUM_TAIL_SEGS;
|
||||
uint32_t num_perm_elems = num_perm_segs * NUM_ELEMS_PER_SEG;
|
||||
uint32_t num_local_tail_elems_padded = num_perm_segs != 0 ? (uint32_t)NUM_TAIL_ELEMS : num_input_segs * NUM_ELEMS_PER_SEG;
|
||||
uint32_t num_rounds = ku::ceil_div(num_local_tail_elems_padded + num_perm_elems, (uint32_t)NUM_ELEMS_PER_ROUND);
|
||||
uint32_t num_init_rounds = min(num_rounds, (uint32_t)NUM_INIT_ROUNDS_MAX);
|
||||
ValueT *init_buf = (ValueT*)smem.incoming_topk_pairs; // aliased
|
||||
|
||||
uint32_t perm_len = max(num_perm_segs, 1u);
|
||||
uint32_t perm_mul = PERM_MUL_PRIME % perm_len;
|
||||
static_assert(NUM_SEGS_PER_ROUND == NUM_SEGS_PER_ISSUE_WARP * NUM_ISSUE_WARPS);
|
||||
__shared__ bool should_select_whole_bucket_shared;
|
||||
|
||||
auto find_pivot_in_histogram = [&](uint32_t topk, const uint32_t *bucket_counter) {
|
||||
bool should_select_whole_bucket =
|
||||
Base::template find_pivot_in_histogram<true>(smem, bucket_counter, topk, lane_idx);
|
||||
if (threadIdx.x == 0) {
|
||||
should_select_whole_bucket_shared = should_select_whole_bucket;
|
||||
}
|
||||
};
|
||||
// First radix pass: histogram the most significant byte of each distorted
|
||||
// fp32 value. Used to locate the top-k bucket prefix.
|
||||
uint32_t hist_base = cute::cast_smem_ptr_to_uint(smem.reconstruct_bucket_counter[0]);
|
||||
// mad forms the bucket address and red.shared.add does the increment: no return value, one address add.
|
||||
auto histogram_radix_msb_one = [&](uint32_t value_bits) {
|
||||
uint32_t addr;
|
||||
asm volatile ("mad.lo.u32 %0, %1, 4, %2; red.shared.add.u32 [%0], 1;"
|
||||
: "=r"(addr)
|
||||
: "r"(topk_select_common::distort(value_bits) >> 24), "r"(hist_base)
|
||||
: "memory");
|
||||
};
|
||||
auto histogram_radix_msb = [&]<uint32_t N>(const uint32_t (&vals)[N], uint32_t m_end) {
|
||||
CUTE_UNROLL
|
||||
for (uint32_t i = 0; i < N; i++) {
|
||||
if (i == m_end) break;
|
||||
histogram_radix_msb_one(vals[i]);
|
||||
}
|
||||
};
|
||||
struct PivotAndQuota {
|
||||
uint32_t pivot_value_bits;
|
||||
uint32_t start_pos_in_collector;
|
||||
uint32_t eq_quota;
|
||||
uint32_t cnt_nan;
|
||||
};
|
||||
|
||||
auto compute_pivot_and_quota = [&](uint32_t topk, const auto &for_each_value) -> PivotAndQuota {
|
||||
if (warp_idx == 0) {
|
||||
find_pivot_in_histogram(topk, smem.reconstruct_bucket_counter[0]);
|
||||
}
|
||||
__syncthreads();
|
||||
// `pivot_prefix` tracks the pivot's leading bits in the *undistorted* (raw) domain.
|
||||
// Inside a round every element that survived the previous round must has the pivot's sign, so the distortion rule (to xor what) is the same.
|
||||
// So we can optimize scanning & picking & distorting into "comparing the raw leading bits" and "xor-ing `neg_mask`"
|
||||
uint32_t pivot_prefix_dist = smem.reconstruct_pivot_bucket; // distorted top 8 bits
|
||||
bool pivot_negative = pivot_prefix_dist < 0x80u; // distorted >= 0x80 <=> positive
|
||||
uint32_t neg_mask = pivot_negative ? 0xFFu : 0u;
|
||||
uint32_t pivot_prefix = pivot_prefix_dist ^ (pivot_negative ? 0xFFu : 0x80u);
|
||||
uint32_t num_elem_should_select_in_pivot_bucket = smem.reconstruct_num_should_select;
|
||||
bool should_select_whole_bucket = should_select_whole_bucket_shared; // whether round 1 is already a clean boundary
|
||||
uint32_t rounds_done = 1;
|
||||
|
||||
// Early exit: once topk lands exactly on a bucket's lower edge, the pivot is that bucket's floor.
|
||||
CUTE_UNROLL
|
||||
for (uint32_t r = 1; r < 4; ++r) {
|
||||
if (should_select_whole_bucket) break;
|
||||
uint32_t *dst = smem.reconstruct_bucket_counter[r & 1];
|
||||
uint32_t *clr = smem.reconstruct_bucket_counter[(r & 1) ^ 1];
|
||||
uint32_t dst_base = cute::cast_smem_ptr_to_uint(dst);
|
||||
for_each_value([&](uint32_t value_bits) {
|
||||
// Elements whose prefix differs from the pivot still add, but to this warp's own sink slot. This let PTXAS generate ATOMS.INC instructions which is faster
|
||||
bool match = (value_bits >> (32 - 8 * r)) == pivot_prefix;
|
||||
// bfe extracts the next raw byte in one instruction (ptxas otherwise emits shf + lop3)
|
||||
uint32_t bucket_raw, bucket;
|
||||
asm volatile ("bfe.u32 %0, %1, %2, 8;" : "=r"(bucket_raw) : "r"(value_bits), "r"(24 - 8 * r));
|
||||
bucket = match ? (bucket_raw ^ neg_mask) : NUM_RECONSTRUCT_BUCKETS;
|
||||
uint32_t addr;
|
||||
asm volatile ("mad.lo.u32 %0, %1, 4, %2; red.shared.add.u32 [%0], 1;"
|
||||
: "=r"(addr) : "r"(bucket), "r"(dst_base) : "memory");
|
||||
});
|
||||
constexpr uint32_t NUM_BUCKET_CLEAR_128b = NUM_RECONSTRUCT_BUCKETS / 4;
|
||||
static_assert(NUM_THREADS >= NUM_BUCKET_CLEAR_128b);
|
||||
if (threadIdx.x < NUM_BUCKET_CLEAR_128b) {
|
||||
reinterpret_cast<__int128_t*>(clr)[threadIdx.x] = (__int128_t)0ull;
|
||||
}
|
||||
__syncthreads();
|
||||
if (warp_idx == 0) {
|
||||
find_pivot_in_histogram(num_elem_should_select_in_pivot_bucket, dst);
|
||||
}
|
||||
__syncthreads();
|
||||
// The histogram is keyed by distorted bytes, so undo the sign map on the bucket id.
|
||||
pivot_prefix = (pivot_prefix << 8) | (smem.reconstruct_pivot_bucket ^ neg_mask);
|
||||
num_elem_should_select_in_pivot_bucket = smem.reconstruct_num_should_select;
|
||||
should_select_whole_bucket = should_select_whole_bucket_shared;
|
||||
rounds_done++;
|
||||
}
|
||||
// Take the lowest element in the selected bucket
|
||||
// Be careful that `pivot` may be negative, and we may have to fill the lower bits with 1
|
||||
uint32_t pivot_low_shift = (4 - rounds_done) * 8;
|
||||
uint32_t pivot_value_bits = pivot_prefix << pivot_low_shift;
|
||||
if (pivot_negative) {
|
||||
pivot_value_bits |= (1u << pivot_low_shift) - 1u;
|
||||
}
|
||||
float pivot_value = __uint_as_float(pivot_value_bits);
|
||||
|
||||
// set.xx.f32.f32 yields 1.0/0.0, so the counts accumulate on the FP pipe instead of the
|
||||
// (busiest) integer pipe; the counts stay exact because FP32 can represent every integer within 0 ~ 2**23
|
||||
float gt_accum = 0.0f, eq_accum = 0.0f;
|
||||
float nan_accum = 0.0f;
|
||||
for_each_value([&](uint32_t value_bits) {
|
||||
float v = __uint_as_float(value_bits);
|
||||
asm volatile (
|
||||
"{\n"
|
||||
".reg .f32 g, e;\n"
|
||||
"set.gt.f32.f32 g, %3, %4;\n"
|
||||
"add.f32 %0, %0, g;\n"
|
||||
"set.eq.f32.f32 e, %3, %4;\n"
|
||||
"add.f32 %1, %1, e;\n"
|
||||
"min.NaN.f32 %2, %2, %3;\n" // We use `min.NaN` for NaN detection
|
||||
"}\n"
|
||||
: "+f"(gt_accum), "+f"(eq_accum), "+f"(nan_accum)
|
||||
: "f"(v), "f"(pivot_value));
|
||||
});
|
||||
uint32_t cnt_gt = (uint32_t)gt_accum;
|
||||
uint32_t cnt_eq = (uint32_t)eq_accum;
|
||||
float nan_flag;
|
||||
asm ("set.nan.f32.f32 %0, %1, %1;" : "=f"(nan_flag) : "f"(nan_accum));
|
||||
uint32_t cnt_nan = (uint32_t)nan_flag;
|
||||
|
||||
static_assert(NUM_WARPS <= NUM_RECONSTRUCT_BUCKETS);
|
||||
uint32_t *eq_pass_warp_cnt = smem.reconstruct_bucket_counter[0];
|
||||
auto eqgt = Base::compute_equal_quota_and_prefix(cnt_gt, cnt_eq, topk, warp_idx, lane_idx, eq_pass_warp_cnt);
|
||||
return {pivot_value_bits, eqgt.start_pos_in_collector, eqgt.eq_quota, cnt_nan};
|
||||
};
|
||||
|
||||
auto reconstruct = [&](uint32_t cur_extra_len) -> uint32_t {
|
||||
uint32_t tid = threadIdx.x;
|
||||
Base::clear_reconstruct_histograms(smem, tid);
|
||||
|
||||
uint32_t padded_extra_len = (cur_extra_len + 1u) & ~1u;
|
||||
if (warp_idx == 0 && (cur_extra_len & 1u)) {
|
||||
smem.incoming_topk_pairs[cur_extra_len] = PLACEHOLDER_PAIR;
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
uint32_t num_128b = (MAX_TOPK + padded_extra_len) / 2;
|
||||
uint32_t num_128b_per_thread = ((num_128b + NUM_THREADS - 1) / NUM_THREADS) | 1u; // force odd => bank-conflict-free smem gathers
|
||||
uint32_t b128_base = tid * num_128b_per_thread;
|
||||
uint32_t num_my_128b = b128_base < num_128b ? min(num_128b_per_thread, num_128b - b128_base) : 0u;
|
||||
|
||||
auto b128_to_pair_addr = [&](uint32_t u) -> uint32_t {
|
||||
// The first MAX_TOPK/2 b128 words live in the current candidate buffer, the rest in the incoming region
|
||||
return u < MAX_TOPK / 2
|
||||
? cute::cast_smem_ptr_to_uint(smem.surviving_topk_pairs[survivor_buf_idx] + 2 * u)
|
||||
: cute::cast_smem_ptr_to_uint(smem.incoming_topk_pairs + (2 * u - MAX_TOPK));
|
||||
};
|
||||
|
||||
// One b128 word is 16 bytes = 2 pairs {index0, value0, index1, value1};
|
||||
constexpr uint32_t NUM_128b_MAX = (MAX_TOPK + RECONSTRUCT_THRESHOLD + NUM_ELEMS_PER_ROUND) / 2;
|
||||
constexpr uint32_t NUM_128b_PER_THREAD = ((NUM_128b_MAX + NUM_THREADS - 1) / NUM_THREADS) | 1u; // | 1u since we may | 1u when generating `num_128b_per_thread`
|
||||
auto for_each_value = [&](const auto &fn) {
|
||||
CUTE_UNROLL
|
||||
for (uint32_t i = 0; i < NUM_128b_PER_THREAD; i++) {
|
||||
if (i == num_my_128b) break;
|
||||
uint32_t pair2[4];
|
||||
topk_select_common::ld_shared<4>(pair2, (const uint32_t*)b128_to_pair_addr(b128_base + i));
|
||||
fn(pair2[1]);
|
||||
fn(pair2[3]);
|
||||
}
|
||||
};
|
||||
|
||||
for_each_value(histogram_radix_msb_one);
|
||||
__syncthreads();
|
||||
|
||||
uint32_t topk = args.topk;
|
||||
auto [pivot_value_bits, start_pos_in_collector, eq_quota, cnt_nan] = compute_pivot_and_quota(topk, for_each_value);
|
||||
// Every NaN the main loop collected is part of the buffer this census just walked (the hit test
|
||||
// is `.gtu`, so NaN always becomes an incomer); the CTA-wide OR happens at the end.
|
||||
have_nan |= cnt_nan != 0;
|
||||
|
||||
{
|
||||
uint32_t out_ptr = cute::cast_smem_ptr_to_uint(smem.surviving_topk_pairs[survivor_buf_idx ^ 1]) + start_pos_in_collector * (uint32_t)sizeof(uint64_t);
|
||||
float pivot_value = __uint_as_float(pivot_value_bits);
|
||||
CUTE_UNROLL
|
||||
for (uint32_t i = 0; i < NUM_128b_PER_THREAD; i++) {
|
||||
if (i == num_my_128b) break;
|
||||
uint32_t pair2[4];
|
||||
topk_select_common::ld_shared<4>(pair2, (const uint32_t*)b128_to_pair_addr(b128_base + i));
|
||||
float v0 = __uint_as_float(pair2[1]);
|
||||
float v1 = __uint_as_float(pair2[3]);
|
||||
bool t0 = v0 == pivot_value && eq_quota != 0;
|
||||
eq_quota -= t0;
|
||||
if (v0 > pivot_value || t0) { append_pair_at(out_ptr, pair2[0], pair2[1]); }
|
||||
bool t1 = v1 == pivot_value && eq_quota != 0;
|
||||
eq_quota -= t1;
|
||||
if (v1 > pivot_value || t1) { append_pair_at(out_ptr, pair2[2], pair2[3]); }
|
||||
}
|
||||
}
|
||||
|
||||
survivor_buf_idx ^= 1;
|
||||
return pivot_value_bits;
|
||||
};
|
||||
|
||||
uint32_t tma_permuted_segment_stride = NUM_ISSUE_WARPS * perm_mul % perm_len;
|
||||
uint32_t next_tma_permuted_segment = Base::get_permuted_seg_idx(warp_idx, perm_len, perm_mul);
|
||||
auto issue_tma_loads_for_round = [&]<bool IS_INIT, bool HAVE_TAIL>(uint32_t idx) {
|
||||
Base::template issue_tma_loads_for_round<IS_INIT, HAVE_TAIL>(
|
||||
smem, tma_params.tensor_map, batch_idx, end_vocab_idx,
|
||||
num_perm_segs, idx, next_tma_permuted_segment, tma_permuted_segment_stride, warp_idx);
|
||||
};
|
||||
|
||||
if (cute::elect_one_sync()) {
|
||||
issue_tma_loads_for_round.template operator()<true, true>(0);
|
||||
CUTE_UNROLL
|
||||
for (uint32_t i = 1; i < NUM_INIT_ROUNDS_MAX; i++) {
|
||||
if (i < num_init_rounds) {
|
||||
issue_tma_loads_for_round.template operator()<true, false>(i);
|
||||
}
|
||||
}
|
||||
{
|
||||
uint32_t num_main_prefetch = num_rounds - num_init_rounds;
|
||||
CUTE_UNROLL
|
||||
for (uint32_t i = 0; i < TMA_PREFETCH_DEPTH; i++) {
|
||||
if (i < num_main_prefetch) {
|
||||
issue_tma_loads_for_round.template operator()<false, false>(i);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
{ // INIT
|
||||
uint32_t num_elems_in_init_window_padded = num_local_tail_elems_padded + min(num_perm_segs, (uint32_t)NUM_PERM_SEGS_IN_INIT_WINDOW) * NUM_ELEMS_PER_SEG;
|
||||
uint32_t num_128b_in_init_window_padded = num_elems_in_init_window_padded / NUM_ELEMS_PER_128b;
|
||||
uint32_t cnt_floor = num_128b_in_init_window_padded / NUM_THREADS;
|
||||
uint32_t cnt_rem = num_128b_in_init_window_padded % NUM_THREADS;
|
||||
uint32_t my_elem_start_idx = (threadIdx.x * cnt_floor + min(threadIdx.x, cnt_rem)) * NUM_ELEMS_PER_128b;
|
||||
uint32_t num_my_elems = NUM_UINT32_PER_128b * cnt_floor + (threadIdx.x < cnt_rem ? NUM_UINT32_PER_128b : 0u);
|
||||
uint32_t num_local_tail_elems = end_vocab_idx - num_perm_elems;
|
||||
|
||||
if (num_perm_segs != 0) {
|
||||
Base::fill_padded_tail_segments(init_buf, end_vocab_idx - num_perm_elems);
|
||||
__syncthreads();
|
||||
}
|
||||
|
||||
CUTE_UNROLL
|
||||
for (uint32_t i = 0; i < NUM_INIT_ROUNDS_MAX; i++) {
|
||||
if (i == num_init_rounds) break;
|
||||
smem.init_full_bar[i].wait(0);
|
||||
if (num_local_tail_elems % NUM_ELEMS_PER_SEG != 0 && i == num_local_tail_elems / NUM_ELEMS_PER_ROUND) {
|
||||
uint32_t box_end = ku::ceil_div(num_local_tail_elems, (uint32_t)NUM_ELEMS_PER_SEG) * NUM_ELEMS_PER_SEG;
|
||||
for (uint32_t e = num_local_tail_elems + threadIdx.x; e < box_end; e += NUM_THREADS) {
|
||||
init_buf[Base::sw_elem(e)] = __uint_as_float(PLACEHOLDER);
|
||||
}
|
||||
__syncthreads();
|
||||
}
|
||||
uint32_t my_values[4 * NUM_128b_PER_THREAD_PER_ROUND];
|
||||
uint32_t num_my_values = 0;
|
||||
CUTE_UNROLL
|
||||
for (uint32_t k = 0; k < NUM_128b_PER_THREAD_PER_ROUND; ++k) {
|
||||
uint32_t u = threadIdx.x + (i * NUM_128b_PER_THREAD_PER_ROUND + k) * NUM_THREADS;
|
||||
if (u < num_128b_in_init_window_padded) {
|
||||
topk_select_common::ld_shared<4>(my_values + k * 4, (const uint32_t*)(init_buf + Base::sw_b128(u) * NUM_ELEMS_PER_128b));
|
||||
num_my_values = 4 * (k + 1);
|
||||
}
|
||||
}
|
||||
histogram_radix_msb(my_values, num_my_values);
|
||||
}
|
||||
|
||||
uint32_t init_values[NUM_UINT32_IN_INIT_WINDOW_PER_THREAD];
|
||||
CUTE_UNROLL
|
||||
for (uint32_t j = 0; j < NUM_128b_INIT_PER_THREAD; ++j) {
|
||||
if (j * NUM_UINT32_PER_128b == num_my_elems) break;
|
||||
uint32_t u0 = my_elem_start_idx / NUM_ELEMS_PER_128b + j;
|
||||
topk_select_common::ld_shared<4>(init_values + j * NUM_UINT32_PER_128b, (const uint32_t*)(init_buf + Base::sw_b128(u0) * NUM_ELEMS_PER_128b));
|
||||
}
|
||||
|
||||
__syncthreads(); // publish the round-1 histogram; also the last read of init_buf
|
||||
|
||||
static_assert(NUM_ELEMS_IN_INIT_WINDOW <= 0xFFFF);
|
||||
|
||||
auto for_each_init_value = [&](const auto &fn) {
|
||||
CUTE_UNROLL
|
||||
for (uint32_t i = 0; i < NUM_UINT32_IN_INIT_WINDOW_PER_THREAD; i++) {
|
||||
if (i == num_my_elems) break;
|
||||
fn(init_values[i]);
|
||||
}
|
||||
};
|
||||
auto [pivot_value_bits, start_pos_in_collector, eq_quota, cnt_nan] = compute_pivot_and_quota(args.topk, for_each_init_value);
|
||||
// The init census walks the thread's whole slice of the window, i.e. every element of the init
|
||||
// window exactly once, so NaNs inside the window are caught here.
|
||||
have_nan |= cnt_nan != 0;
|
||||
|
||||
{
|
||||
uint32_t out_ptr = cute::cast_smem_ptr_to_uint(smem.surviving_topk_pairs[0]) + start_pos_in_collector * (uint32_t)sizeof(uint64_t);
|
||||
float pivot_value = __uint_as_float(pivot_value_bits);
|
||||
uint32_t s0 = (max(my_elem_start_idx, num_local_tail_elems_padded) - num_local_tail_elems_padded) / NUM_ELEMS_PER_SEG;
|
||||
uint32_t perm_state = Base::get_permuted_seg_idx(s0, perm_len, perm_mul);
|
||||
float b128_base_f = 0.0f;
|
||||
CUTE_UNROLL
|
||||
for (uint32_t i = 0; i < NUM_UINT32_IN_INIT_WINDOW_PER_THREAD; i += 2) {
|
||||
if (i == num_my_elems) break;
|
||||
if (i % NUM_ELEMS_PER_128b == 0) {
|
||||
uint32_t g_u = my_elem_start_idx + i; // fp32: word index == element index
|
||||
if (g_u < num_local_tail_elems_padded) {
|
||||
b128_base_f = __uint_as_float(num_perm_elems + g_u);
|
||||
} else {
|
||||
b128_base_f = __uint_as_float(perm_state * NUM_ELEMS_PER_SEG + g_u % NUM_ELEMS_PER_SEG);
|
||||
if (g_u % NUM_ELEMS_PER_SEG == NUM_ELEMS_PER_SEG - NUM_ELEMS_PER_128b) {
|
||||
Base::advance_perm_state(perm_state, perm_mul, perm_len);
|
||||
}
|
||||
}
|
||||
}
|
||||
// fp32-subnormal FP-pipe int-add (see file-head): index = base + element offset
|
||||
uint32_t index0 = __float_as_uint(b128_base_f + __uint_as_float(i % NUM_ELEMS_PER_128b));
|
||||
uint32_t index1 = __float_as_uint(b128_base_f + __uint_as_float(i % NUM_ELEMS_PER_128b + 1));
|
||||
float v0 = __uint_as_float(init_values[i]);
|
||||
float v1 = __uint_as_float(init_values[i + 1]);
|
||||
bool t0 = v0 == pivot_value && eq_quota != 0;
|
||||
eq_quota -= t0;
|
||||
if (v0 > pivot_value || t0) { append_pair_at(out_ptr, index0, init_values[i]); }
|
||||
// the quota is re-tested after element 0's decrement
|
||||
bool t1 = v1 == pivot_value && eq_quota != 0;
|
||||
eq_quota -= t1;
|
||||
if (v1 > pivot_value || t1) { append_pair_at(out_ptr, index1, init_values[i + 1]); }
|
||||
}
|
||||
}
|
||||
|
||||
threshold_bits = pivot_value_bits;
|
||||
__syncthreads();
|
||||
}
|
||||
|
||||
uint32_t linear_segment_start =
|
||||
num_init_rounds * NUM_SEGS_PER_ROUND
|
||||
- NUM_TAIL_SEGS
|
||||
+ warp_idx;
|
||||
uint32_t current_permuted_segment = Base::get_permuted_seg_idx(linear_segment_start, perm_len, perm_mul);
|
||||
uint32_t permuted_segment_stride_per_round = NUM_SEGS_PER_ROUND % perm_len * perm_mul % perm_len;
|
||||
|
||||
uint32_t num_main_rounds = num_rounds - num_init_rounds;
|
||||
// Track the TMA buffer index and its phase incrementally instead of dividing every round.
|
||||
uint32_t tma_buf_idx = 0;
|
||||
uint32_t tma_buf_phase = 0;
|
||||
for (uint32_t main_round_idx = 0; main_round_idx < num_main_rounds; ++main_round_idx) {
|
||||
// This thread's contiguous chunk starts at logical_elem_offset in this round.
|
||||
uint32_t logical_elem_offset = threadIdx.x * NUM_ELEMS_PER_THREAD_PER_ROUND;
|
||||
// Offset inside the 512-element segment; used later to rebuild original indices.
|
||||
uint32_t offset_in_segment = logical_elem_offset % NUM_ELEMS_PER_SEG;
|
||||
// TMA stores the round data in a swizzled layout. swizzle_mask describes how the
|
||||
// logical element offset is mapped to the actual shared-memory offset.
|
||||
uint32_t swizzle_mask = Base::sw_msk(logical_elem_offset);
|
||||
uint32_t chunk_swizzle_mask = swizzle_mask & ELEMS_PER_THREAD_PER_ROUND_MASK;
|
||||
uint32_t smem_read_offset = logical_elem_offset ^ (swizzle_mask & ~ELEMS_PER_THREAD_PER_ROUND_MASK);
|
||||
|
||||
if (main_round_idx + TMA_PREFETCH_DEPTH < num_main_rounds) {
|
||||
if (cute::elect_one_sync()) {
|
||||
issue_tma_loads_for_round.template operator()<false, false>(main_round_idx + TMA_PREFETCH_DEPTH);
|
||||
}
|
||||
}
|
||||
|
||||
bool is_warp_active = true;
|
||||
if constexpr (HAS_PARTIAL_ROUNDS) {
|
||||
is_warp_active = (num_init_rounds + main_round_idx) * NUM_SEGS_PER_ROUND + warp_idx < NUM_TAIL_SEGS + num_perm_segs;
|
||||
}
|
||||
|
||||
const ValueT *buf = smem.tma_load_buf[tma_buf_idx];
|
||||
// One bit per element of this thread's slice, in element order.
|
||||
// Hits are rare, so appending through the set bits of this mask is faster than testing + storing every element again.
|
||||
static_assert(NUM_ELEMS_PER_THREAD_PER_ROUND <= 32); // hit_mask is a uint32_t
|
||||
uint32_t hit_mask = 0;
|
||||
if (is_warp_active) {
|
||||
smem.tma_load_full_bar[tma_buf_idx].wait(tma_buf_phase);
|
||||
ValueT values[NUM_ELEMS_PER_THREAD_PER_ROUND];
|
||||
Base::template load_swizzled_slice<NUM_128b_PER_THREAD_PER_ROUND>(values, buf, smem_read_offset, chunk_swizzle_mask);
|
||||
|
||||
float threshold = __uint_as_float(threshold_bits);
|
||||
CUTE_UNROLL
|
||||
for (uint32_t j = 0; j < NUM_ELEMS_PER_THREAD_PER_ROUND; ++j) {
|
||||
asm volatile (
|
||||
"{\n"
|
||||
".reg .pred p;\n"
|
||||
"setp.gtu.f32 p, %1, %2;\n" // .gtu: NaN always counts as a hit, so every NaN reaches the incoming buffer and the census reports it
|
||||
"@p or.b32 %0, %0, %3;\n"
|
||||
"}\n"
|
||||
: "+r"(hit_mask)
|
||||
: "f"((float)values[j]), "f"(threshold), "r"(1u << j));
|
||||
}
|
||||
}
|
||||
uint32_t num_new_incomers = __popc(hit_mask);
|
||||
|
||||
uint32_t warp_total_hits = __reduce_add_sync(0xFFFFFFFF, num_new_incomers);
|
||||
if (lane_idx == 0) {
|
||||
smem.warp_cnt[warp_idx] = warp_total_hits;
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
uint32_t seg_elem_base = current_permuted_segment * NUM_ELEMS_PER_SEG + offset_in_segment;
|
||||
// Element e of this thread's slice lives at smem_read_offset + (e ^ chunk_swizzle_mask)
|
||||
uint32_t elem_addr_base = cute::cast_smem_ptr_to_uint(buf + smem_read_offset);
|
||||
|
||||
// Start the first hit's smem load before the count exchange below, so that its latency
|
||||
// (and the barrier wait) overlaps with them instead of delaying the first store.
|
||||
uint32_t first_hit_e = hit_mask != 0 ? __ffs(hit_mask) - 1u : 0u;
|
||||
uint32_t first_hit_val = 0;
|
||||
if (is_warp_active && warp_total_hits != 0) {
|
||||
asm volatile ("ld.shared.u32 %0, [%1];" : "=r"(first_hit_val) : "r"(elem_addr_base + ((first_hit_e ^ chunk_swizzle_mask) << 2)));
|
||||
}
|
||||
|
||||
static_assert(NUM_WARPS <= 32);
|
||||
uint32_t stored_warp_hits = lane_idx < NUM_WARPS ? smem.warp_cnt[lane_idx] : 0u;
|
||||
uint32_t num_total_hits_in_this_round = __reduce_add_sync(0xFFFFFFFF, stored_warp_hits);
|
||||
|
||||
uint32_t dst_slot =
|
||||
num_incomers +
|
||||
__reduce_add_sync(0xFFFFFFFF, lane_idx < warp_idx ? stored_warp_hits : 0u) +
|
||||
warp_level_exclusive_prefix_sum(num_new_incomers, lane_idx);
|
||||
|
||||
if (is_warp_active && warp_total_hits != 0) {
|
||||
uint32_t dst_ptr = cute::cast_smem_ptr_to_uint(smem.incoming_topk_pairs) + dst_slot * (uint32_t)sizeof(uint64_t);
|
||||
if (hit_mask != 0) {
|
||||
asm volatile ("st.shared.v2.u32 [%0], {%1, %2};" :: "r"(dst_ptr), "r"(seg_elem_base + first_hit_e), "r"(first_hit_val) : "memory");
|
||||
dst_ptr += (uint32_t)sizeof(uint64_t);
|
||||
}
|
||||
uint32_t mask = hit_mask & (hit_mask - 1u);
|
||||
while (mask != 0) {
|
||||
uint32_t e = __ffs(mask) - 1u;
|
||||
mask &= mask - 1u;
|
||||
uint32_t val_word;
|
||||
asm volatile ("ld.shared.u32 %0, [%1];" : "=r"(val_word) : "r"(elem_addr_base + ((e ^ chunk_swizzle_mask) << 2)));
|
||||
asm volatile ("st.shared.v2.u32 [%0], {%1, %2};" :: "r"(dst_ptr), "r"(seg_elem_base + e), "r"(val_word) : "memory");
|
||||
dst_ptr += (uint32_t)sizeof(uint64_t);
|
||||
}
|
||||
}
|
||||
Base::advance_perm_state(current_permuted_segment, permuted_segment_stride_per_round, perm_len);
|
||||
|
||||
num_incomers += num_total_hits_in_this_round;
|
||||
__syncthreads();
|
||||
|
||||
if (++tma_buf_idx == NUM_TMA_LOAD_BUFS) {
|
||||
tma_buf_idx = 0;
|
||||
tma_buf_phase ^= 1u;
|
||||
}
|
||||
|
||||
if (num_incomers >= RECONSTRUCT_THRESHOLD) {
|
||||
threshold_bits = reconstruct(num_incomers);
|
||||
num_incomers = 0;
|
||||
}
|
||||
}
|
||||
|
||||
if (num_incomers > 0) {
|
||||
reconstruct(num_incomers);
|
||||
num_incomers = 0;
|
||||
}
|
||||
|
||||
have_nan = __syncthreads_or(have_nan) != 0;
|
||||
if (have_nan) {
|
||||
Base::take_action_when_have_nan(args, batch_idx);
|
||||
return;
|
||||
}
|
||||
|
||||
Base::template stage_output_and_epilogue<Config::sorted_index>(
|
||||
smem, *reinterpret_cast<typename EpilogueT::BlockRadixSortTempStorageT*>(smem.incoming_topk_pairs), args,
|
||||
batch_idx, end_vocab_idx, survivor_buf_idx, warp_idx, lane_idx);
|
||||
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
|
||||
template<typename Kernel>
|
||||
__launch_bounds__(Kernel::NUM_THREADS, Kernel::TARGET_OCCUPANCY, 1)
|
||||
__global__ void topk_kernel(__grid_constant__ const TopkSelectArgs args, __grid_constant__ const typename Kernel::TmaParams tma_params) {
|
||||
Kernel::topk_select_kernel_devfunc(args, tma_params);
|
||||
}
|
||||
|
||||
template<typename Config>
|
||||
void run_topk_select_kernel(const TopkSelectArgs &args) {
|
||||
KU_ASSERT(args.sorted_value == Config::sorted_value, "Dispatch failure");
|
||||
KU_ASSERT(args.sorted_index == Config::sorted_index, "Dispatch failure");
|
||||
KU_ASSERT(args.return_value == Config::return_value, "Dispatch failure");
|
||||
static_assert(cute::is_same_v<typename Config::ValueT, float>);
|
||||
KU_ASSERT(args.vocab_size < MAX_VOCAB_SIZE, "`vocab_size` is too big");
|
||||
|
||||
using Kernel = TopkSelectKernelFP32<Config>;
|
||||
KU_ASSERT(args.topk <= Kernel::MAX_TOPK, "topk is too large. Maximum allowed: %d\n", Kernel::MAX_TOPK);
|
||||
static_assert(INPUT_STRIDE_ALIGNMENT_REQUIREMENT % 16 == 0);
|
||||
|
||||
auto kernel = topk_kernel<Kernel>;
|
||||
constexpr size_t smem_size = sizeof(typename Kernel::SharedMemoryPlanFP32);
|
||||
KU_ASSERT(smem_size * Kernel::TARGET_OCCUPANCY <= args.shared_memory_size_per_sm);
|
||||
KU_CUDA_CHECK(cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, smem_size));
|
||||
|
||||
KU_ASSERT(args.stride_input_batch % 4 == 0, "stride_input_batch must be 16B-aligned");
|
||||
typename Kernel::TmaParams tma_params = {Kernel::make_topk_tensor_map(args)};
|
||||
|
||||
ku::launch_kernel(ku::KernelLaunchConfig {
|
||||
dim3(args.batch_size),
|
||||
dim3(Kernel::NUM_THREADS),
|
||||
smem_size,
|
||||
args.stream
|
||||
}, kernel, args, tma_params);
|
||||
}
|
||||
|
||||
} // topk_select_fp32
|
||||
@@ -0,0 +1,39 @@
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <cuda_runtime_api.h>
|
||||
|
||||
static constexpr uint32_t INPUT_STRIDE_ALIGNMENT_REQUIREMENT = 1024; // In number of bytes
|
||||
static constexpr uint32_t OUTPUT_STRIDE_ALIGNMENT_REQUIREMENT = 32; // In number of bytes
|
||||
|
||||
static constexpr uint32_t MAX_INT_ADDITION_RANGE_BY_FP32_SIMULATION = 1u << 23;
|
||||
static constexpr uint32_t MAX_VOCAB_SIZE = 1u << 23;
|
||||
static_assert(MAX_VOCAB_SIZE <= MAX_INT_ADDITION_RANGE_BY_FP32_SIMULATION);
|
||||
|
||||
struct TopkSelectArgs {
|
||||
uint32_t batch_size;
|
||||
uint32_t vocab_size;
|
||||
uint32_t topk;
|
||||
|
||||
void* input;
|
||||
void* output_value;
|
||||
void* output_index;
|
||||
int* begin_ptr;
|
||||
int* end_ptr;
|
||||
int* output_idx_offset;
|
||||
|
||||
// All strides are in number of elements, not bytes
|
||||
uint64_t stride_input_batch;
|
||||
uint64_t stride_output_value_batch;
|
||||
uint64_t stride_output_index_batch;
|
||||
|
||||
bool sorted_value;
|
||||
bool sorted_index;
|
||||
bool return_value;
|
||||
int idx_oob_fill_value;
|
||||
float value_oob_fill_value;
|
||||
bool abort_when_nan_found;
|
||||
|
||||
uint64_t shared_memory_size_per_sm;
|
||||
cudaStream_t stream;
|
||||
};
|
||||
@@ -8,6 +8,8 @@
|
||||
|
||||
#include <sgl_kernel/utils.cuh>
|
||||
|
||||
#include <tvm/ffi/container/tensor.h>
|
||||
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <utility>
|
||||
@@ -67,8 +69,13 @@ inline void* get_device_accessible_ptr(const tvm::ffi::TensorView& tensor) {
|
||||
return device_ptr;
|
||||
}
|
||||
|
||||
namespace details {
|
||||
|
||||
/**
|
||||
* \brief Memo for a per-device driver query, keyed by device ordinal.
|
||||
*
|
||||
* \tparam T Cached value type.
|
||||
* \tparam kDefault The "not queried yet" sentinel. A query that can legally
|
||||
* return it would be re-run on every call, so pick one it cannot.
|
||||
*/
|
||||
template <typename T, T kDefault>
|
||||
struct DeviceCacheMap {
|
||||
public:
|
||||
@@ -99,8 +106,6 @@ struct DeviceCacheMap {
|
||||
T m_data[kNumStaticMaxDevice];
|
||||
};
|
||||
|
||||
} // namespace details
|
||||
|
||||
// Return the maximum number of active blocks per SM for the given kernel
|
||||
template <typename T>
|
||||
inline auto get_blocks_per_sm(T&& kernel, int32_t block_dim, std::size_t dynamic_smem = 0) -> uint32_t {
|
||||
@@ -111,7 +116,7 @@ inline auto get_blocks_per_sm(T&& kernel, int32_t block_dim, std::size_t dynamic
|
||||
|
||||
// Return the number of SMs for the given device
|
||||
inline auto get_sm_count(int device_id, bool use_cache = true) -> uint32_t {
|
||||
static details::DeviceCacheMap<uint32_t, 0> sm_count_cache;
|
||||
static DeviceCacheMap<uint32_t, 0> sm_count_cache;
|
||||
return sm_count_cache.get_cached(device_id, use_cache, [](int32_t device_id) {
|
||||
int sm_count;
|
||||
CHECK_CUDA(cudaDeviceGetAttribute(&sm_count, cudaDevAttrMultiProcessorCount, device_id));
|
||||
@@ -121,7 +126,7 @@ inline auto get_sm_count(int device_id, bool use_cache = true) -> uint32_t {
|
||||
|
||||
// Return the Major compute capability for the given device
|
||||
inline auto get_cc_major(int device_id, bool use_cache = true) -> int {
|
||||
static details::DeviceCacheMap<int, -1> cc_major_cache;
|
||||
static DeviceCacheMap<int, -1> cc_major_cache;
|
||||
return cc_major_cache.get_cached(device_id, use_cache, [](int32_t device_id) {
|
||||
int cc_major;
|
||||
CHECK_CUDA(cudaDeviceGetAttribute(&cc_major, cudaDevAttrComputeCapabilityMajor, device_id));
|
||||
@@ -131,7 +136,7 @@ inline auto get_cc_major(int device_id, bool use_cache = true) -> int {
|
||||
|
||||
// Return the Minor compute capability for the given device
|
||||
inline auto get_cc_minor(int device_id, bool use_cache = true) -> int {
|
||||
static details::DeviceCacheMap<int, -1> cc_minor_cache;
|
||||
static DeviceCacheMap<int, -1> cc_minor_cache;
|
||||
return cc_minor_cache.get_cached(device_id, use_cache, [](int32_t device_id) {
|
||||
int cc_minor;
|
||||
CHECK_CUDA(cudaDeviceGetAttribute(&cc_minor, cudaDevAttrComputeCapabilityMinor, device_id));
|
||||
@@ -159,6 +164,26 @@ inline auto get_available_dynamic_smem_per_block(T&& kernel, int num_blocks, int
|
||||
return smem_size;
|
||||
}
|
||||
|
||||
/**
|
||||
* \brief Shared memory one block may use on the given device, in bytes.
|
||||
*
|
||||
* \param device_id CUDA device ordinal.
|
||||
* \param opt_in True for the larger limit a kernel reaches by setting
|
||||
* `cudaFuncAttributeMaxDynamicSharedMemorySize`, false for the default
|
||||
* cap that needs no opt-in.
|
||||
* \param use_cache False re-queries the driver instead of reading the memo.
|
||||
*/
|
||||
inline auto get_max_smem_per_block(int device_id, bool opt_in = true, bool use_cache = true) -> std::size_t {
|
||||
/// NOTE: we all know this is no larger than 1MB for all GPU, so use uint32_t
|
||||
static DeviceCacheMap<uint32_t, 0> smem_cache[2];
|
||||
const auto attr = opt_in ? cudaDevAttrMaxSharedMemoryPerBlockOptin : cudaDevAttrMaxSharedMemoryPerBlock;
|
||||
return smem_cache[opt_in].get_cached(device_id, use_cache, [attr](int32_t device_id) {
|
||||
int value = 0;
|
||||
CHECK_CUDA(cudaDeviceGetAttribute(&value, attr, device_id));
|
||||
return value;
|
||||
});
|
||||
}
|
||||
|
||||
} // namespace host::runtime
|
||||
|
||||
} // namespace sglang
|
||||
|
||||
@@ -7,6 +7,7 @@ from sglang.kernels.jit.utils.arch import (
|
||||
override_jit_cuda_arch,
|
||||
)
|
||||
from sglang.kernels.jit.utils.common import (
|
||||
aligned_new_empty,
|
||||
cache_once,
|
||||
empty_sentinel,
|
||||
get_ci_test_range,
|
||||
@@ -23,6 +24,7 @@ from sglang.kernels.jit.utils.compile import (
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"aligned_new_empty",
|
||||
"empty_sentinel",
|
||||
"should_run_full_tests",
|
||||
"get_ci_test_range",
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import functools
|
||||
from typing import Any, Callable, Dict, List, TypeVar
|
||||
from typing import Any, Callable, Dict, List, Sequence, TypeVar
|
||||
|
||||
import torch
|
||||
|
||||
@@ -86,3 +86,28 @@ def lazy_register_class(name: str, init_fn: Callable[[], None]) -> Callable[[T],
|
||||
return cls
|
||||
|
||||
return decorator
|
||||
|
||||
|
||||
def aligned_new_empty(
|
||||
shape: Sequence[int],
|
||||
dtype: torch.dtype,
|
||||
device: torch.device,
|
||||
*,
|
||||
alignment: int,
|
||||
) -> torch.Tensor:
|
||||
"""An uninitialized tensor whose row stride is a multiple of `alignment` bytes.
|
||||
|
||||
The last dimension is padded up and then sliced back off, so the result has
|
||||
the requested shape and a non-contiguous stride whenever padding was needed.
|
||||
Callers are kernels that state an alignment contract on a row address, such
|
||||
as a TMA descriptor or a vectorized load.
|
||||
"""
|
||||
assert len(shape) > 1
|
||||
assert alignment % dtype.itemsize == 0, (
|
||||
"alignment must be a whole number of elements"
|
||||
)
|
||||
step = alignment // dtype.itemsize
|
||||
last_dim = shape[-1]
|
||||
padded_last_dim = (last_dim + step - 1) // step * step
|
||||
aligned_shape = (*shape[:-1], padded_last_dim)
|
||||
return torch.empty(aligned_shape, dtype=dtype, device=device)[..., :last_dim]
|
||||
|
||||
@@ -0,0 +1,320 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, Dict, NamedTuple, Optional, Tuple
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.jit.utils import (
|
||||
KERNEL_PATH,
|
||||
aligned_new_empty,
|
||||
cache_once,
|
||||
load_jit,
|
||||
make_cpp_args,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from tvm_ffi.module import Module
|
||||
|
||||
# Must match INPUT_/OUTPUT_STRIDE_ALIGNMENT_REQUIREMENT in vendor/structs.h; the
|
||||
# kernel's TMA descriptor depends on them and `entry.cuh` rejects a row that
|
||||
# misses one.
|
||||
_INPUT_STRIDE_ALIGNMENT_BYTES = 1024
|
||||
_OUTPUT_STRIDE_ALIGNMENT_BYTES = 32
|
||||
_CLUSTER_MAX_TOPK = 1024
|
||||
_CLUSTER_MAX_BATCH_SIZE = 6
|
||||
_SUPPORTED_CAPABILITIES = ((9, 0), (10, 0), (10, 3))
|
||||
|
||||
|
||||
class _Config(NamedTuple):
|
||||
max_topk: int
|
||||
num_threads: int
|
||||
occupancy: int
|
||||
elements_per_round: int
|
||||
reconstruct_threshold: int
|
||||
tma_buffer_depth: int
|
||||
cluster_size: int = 1
|
||||
|
||||
def to_cpp_class_name(
|
||||
self,
|
||||
value_dtype: torch.dtype,
|
||||
index_dtype: torch.dtype,
|
||||
sorted_value: bool,
|
||||
sorted_index: bool,
|
||||
return_value: bool,
|
||||
) -> str:
|
||||
template_args = make_cpp_args(
|
||||
value_dtype,
|
||||
index_dtype,
|
||||
sorted_value,
|
||||
sorted_index,
|
||||
return_value,
|
||||
self.max_topk,
|
||||
self.num_threads,
|
||||
self.occupancy,
|
||||
self.elements_per_round,
|
||||
self.reconstruct_threshold,
|
||||
self.tma_buffer_depth,
|
||||
512, # elements_per_segment, fixed upstream
|
||||
self.cluster_size,
|
||||
)
|
||||
return f"TopkSelectConfig<{template_args}>"
|
||||
|
||||
|
||||
# Per (value dtype, top-k bucket): the tuning for a single-wave batch and, where
|
||||
# the bucket splits on wave count, the tuning for anything larger. A single
|
||||
# entry means no split, and then only one kernel is instantiated.
|
||||
_NORMAL_CONFIGS: Dict[Tuple[torch.dtype, int], Tuple[_Config, ...]] = {
|
||||
(torch.bfloat16, 512): (
|
||||
_Config(512, 512, 1, 8192, 4096, 5),
|
||||
_Config(512, 256, 2, 4096, 4096, 4),
|
||||
),
|
||||
(torch.bfloat16, 1024): (
|
||||
_Config(1024, 512, 1, 8192, 4096, 5),
|
||||
_Config(1024, 256, 2, 4096, 4096, 3),
|
||||
),
|
||||
(torch.bfloat16, 4096): (_Config(4096, 512, 1, 8192, 4096, 3),),
|
||||
(torch.float32, 512): (_Config(512, 512, 1, 8192, 4096, 3),),
|
||||
(torch.float32, 1024): (_Config(1024, 512, 1, 8192, 4096, 3),),
|
||||
(torch.float32, 4096): (_Config(4096, 256, 1, 4096, 4096, 3),),
|
||||
}
|
||||
|
||||
|
||||
@cache_once
|
||||
def _cluster_tuning(device_capability: Tuple[int, int]) -> Tuple[int, int]:
|
||||
# Return (cluster_size, min_vocab_size)
|
||||
major = device_capability[0]
|
||||
return (8, 128 * 1024) if major == 9 else (16, 512 * 1024)
|
||||
|
||||
|
||||
@cache_once
|
||||
def _jit_deep_select_module(
|
||||
value_dtype: torch.dtype,
|
||||
index_dtype: torch.dtype,
|
||||
sorted_value: bool,
|
||||
sorted_index: bool,
|
||||
return_value: bool,
|
||||
max_topk: int,
|
||||
cluster_size: int,
|
||||
) -> Module:
|
||||
assert value_dtype in (torch.bfloat16, torch.float32)
|
||||
assert index_dtype in (torch.int32, torch.int64)
|
||||
if sorted_value:
|
||||
assert value_dtype == torch.float32
|
||||
assert return_value and not sorted_index
|
||||
if cluster_size == 1:
|
||||
configs = _NORMAL_CONFIGS[(value_dtype, max_topk)]
|
||||
host_dispatch = "TopkNormal"
|
||||
else:
|
||||
# Upstream tunes the cluster kernel for bf16 at max_topk 1024 alone, so
|
||||
# the 512 and 1024 buckets share this module.
|
||||
assert value_dtype == torch.bfloat16
|
||||
assert max_topk == _CLUSTER_MAX_TOPK
|
||||
configs = [_Config(max_topk, 256, 1, 4096, 4096, 16, cluster_size)]
|
||||
host_dispatch = "TopkCluster"
|
||||
classes = [
|
||||
config.to_cpp_class_name(
|
||||
value_dtype=value_dtype,
|
||||
index_dtype=index_dtype,
|
||||
sorted_value=sorted_value,
|
||||
sorted_index=sorted_index,
|
||||
return_value=return_value,
|
||||
)
|
||||
for config in configs
|
||||
]
|
||||
classes = make_cpp_args(*classes)
|
||||
root = (KERNEL_PATH / "csrc" / "deepselect" / "vendor").resolve()
|
||||
return load_jit(
|
||||
"deep_select_topk",
|
||||
# cache only distinct key for better readability
|
||||
*make_cpp_args(
|
||||
value_dtype,
|
||||
index_dtype,
|
||||
sorted_value,
|
||||
sorted_index,
|
||||
return_value,
|
||||
max_topk,
|
||||
cluster_size,
|
||||
),
|
||||
cuda_files=["deepselect/entry.cuh"],
|
||||
cuda_wrappers=[("topk", f"deepselect::{host_dispatch}<{classes}>::topk")],
|
||||
extra_include_paths=[
|
||||
str(root),
|
||||
str(root / "3rdparty" / "kerutils" / "include"),
|
||||
],
|
||||
extra_dependencies=["cutlass"],
|
||||
extra_cuda_cflags=[
|
||||
"--expt-extended-lambda",
|
||||
"--use_fast_math",
|
||||
"--ftz=false",
|
||||
"-Xptxas=--register-usage-level=10",
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def _get_max_topk_bucket(topk: int, use_cluster: bool) -> int:
|
||||
if use_cluster:
|
||||
return _CLUSTER_MAX_TOPK
|
||||
return 512 if topk <= 512 else 1024 if topk <= 1024 else 4096
|
||||
|
||||
|
||||
def get_input_stride_alignment_bytes() -> int:
|
||||
return _INPUT_STRIDE_ALIGNMENT_BYTES
|
||||
|
||||
|
||||
def get_stride_requirement() -> Tuple[int, int]:
|
||||
"""Return the input and output row-stride requirements in bytes."""
|
||||
return _INPUT_STRIDE_ALIGNMENT_BYTES, _OUTPUT_STRIDE_ALIGNMENT_BYTES
|
||||
|
||||
|
||||
def get_deepselect_supported_architectures() -> Tuple[int, ...]:
|
||||
"""Return the CUDA compute capabilities supported by the JIT kernel."""
|
||||
return tuple(major * 10 + minor for major, minor in _SUPPORTED_CAPABILITIES)
|
||||
|
||||
|
||||
def is_deepselect_supported(device=None) -> bool:
|
||||
"""Return whether DeepSelect JIT supports a CUDA device."""
|
||||
if torch.version.cuda is None or not torch.cuda.is_available():
|
||||
return False
|
||||
try:
|
||||
normalized_device = (
|
||||
torch.device("cuda", device)
|
||||
if isinstance(device, int)
|
||||
else (
|
||||
torch.device("cuda", torch.cuda.current_device())
|
||||
if device is None
|
||||
else torch.device(device)
|
||||
)
|
||||
)
|
||||
if normalized_device.type != "cuda":
|
||||
return False
|
||||
return (
|
||||
torch.cuda.get_device_capability(normalized_device)
|
||||
in _SUPPORTED_CAPABILITIES
|
||||
)
|
||||
except (RuntimeError, TypeError, ValueError):
|
||||
return False
|
||||
|
||||
|
||||
def _needs_output_staging(output: torch.Tensor, topk: int) -> bool:
|
||||
return (
|
||||
output.data_ptr() % _OUTPUT_STRIDE_ALIGNMENT_BYTES != 0
|
||||
or topk * output.element_size() % _OUTPUT_STRIDE_ALIGNMENT_BYTES != 0
|
||||
)
|
||||
|
||||
|
||||
def topk(
|
||||
input: torch.Tensor,
|
||||
topk: int,
|
||||
sorted: bool = False,
|
||||
begin: Optional[torch.Tensor] = None,
|
||||
end: Optional[torch.Tensor] = None,
|
||||
indices_type: torch.dtype = torch.int64,
|
||||
sorted_index: bool = False,
|
||||
hint: Optional[torch.Tensor] = None,
|
||||
output_idx: Optional[torch.Tensor] = None,
|
||||
output_idx_offset: Optional[torch.Tensor] = None,
|
||||
idx_oob_fill_value: int = 2147483647,
|
||||
value_oob_fill_value: float = float("-inf"),
|
||||
return_value: bool = True,
|
||||
abort_when_nan_found: bool = True,
|
||||
) -> Tuple[Optional[torch.Tensor], torch.Tensor]:
|
||||
"""Select the largest ``topk`` values of every row of ``input``.
|
||||
|
||||
This follows the public DeepSelect interface. ``end`` is the per-row
|
||||
exclusive valid length; ``begin`` and ``hint`` are reserved but unsupported.
|
||||
|
||||
Returns ``(values, indices)``; ``values`` is None when ``return_value`` is
|
||||
False, which skips writing them and is about 10% faster.
|
||||
"""
|
||||
if hint is not None:
|
||||
raise ValueError("hint is not supported currently")
|
||||
if input.dtype not in (torch.bfloat16, torch.float32):
|
||||
raise RuntimeError("input dtype must be bfloat16 or float32")
|
||||
if indices_type not in (torch.int32, torch.int64):
|
||||
raise RuntimeError("indices_type must be int32 or int64")
|
||||
if output_idx is not None and output_idx.dtype != indices_type:
|
||||
raise ValueError("output_idx dtype must match indices_type")
|
||||
if input.device.type != "cuda":
|
||||
raise RuntimeError("input must be a CUDA tensor")
|
||||
if input.ndim != 2:
|
||||
raise RuntimeError("input must be a 2D tensor")
|
||||
if not 0 < topk <= 4096:
|
||||
raise RuntimeError(f"topk must be in [1, 4096], got {topk}")
|
||||
if sorted and not return_value:
|
||||
raise RuntimeError("return_value must be enabled when sorted is True")
|
||||
if sorted and sorted_index:
|
||||
raise RuntimeError("sorted and sorted_index cannot both be True")
|
||||
if sorted and input.dtype is torch.bfloat16:
|
||||
raise RuntimeError("sorted is only supported for float32 input")
|
||||
|
||||
rows, vocab_size = input.shape
|
||||
device_capability = torch.cuda.get_device_capability(input.device)
|
||||
if device_capability not in _SUPPORTED_CAPABILITIES:
|
||||
major, minor = device_capability
|
||||
raise RuntimeError(f"DeepSelect does not support SM{major}{minor}")
|
||||
cluster_size, cluster_min_vocab_size = _cluster_tuning(device_capability)
|
||||
use_cluster = (
|
||||
input.dtype is torch.bfloat16
|
||||
and rows <= _CLUSTER_MAX_BATCH_SIZE
|
||||
and vocab_size >= cluster_min_vocab_size
|
||||
and topk <= _CLUSTER_MAX_TOPK
|
||||
)
|
||||
values = None
|
||||
if return_value:
|
||||
values = aligned_new_empty(
|
||||
(rows, topk),
|
||||
input.dtype,
|
||||
input.device,
|
||||
alignment=_OUTPUT_STRIDE_ALIGNMENT_BYTES,
|
||||
)
|
||||
if output_idx is None:
|
||||
output_idx = aligned_new_empty(
|
||||
(rows, topk),
|
||||
indices_type,
|
||||
input.device,
|
||||
alignment=_OUTPUT_STRIDE_ALIGNMENT_BYTES,
|
||||
)
|
||||
kernel_output_idx = output_idx
|
||||
else:
|
||||
kernel_output_idx = (
|
||||
aligned_new_empty(
|
||||
(rows, topk),
|
||||
indices_type,
|
||||
input.device,
|
||||
alignment=_OUTPUT_STRIDE_ALIGNMENT_BYTES,
|
||||
)
|
||||
if _needs_output_staging(output_idx, topk)
|
||||
else output_idx
|
||||
)
|
||||
module = _jit_deep_select_module(
|
||||
input.dtype,
|
||||
indices_type,
|
||||
sorted,
|
||||
sorted_index,
|
||||
return_value,
|
||||
_get_max_topk_bucket(topk, use_cluster),
|
||||
cluster_size if use_cluster else 1,
|
||||
)
|
||||
input_storage_bytes = (
|
||||
input.untyped_storage().nbytes() - input.storage_offset() * input.element_size()
|
||||
)
|
||||
module.topk(
|
||||
input,
|
||||
values,
|
||||
output_idx,
|
||||
kernel_output_idx,
|
||||
begin,
|
||||
end,
|
||||
output_idx_offset,
|
||||
topk,
|
||||
input_storage_bytes,
|
||||
idx_oob_fill_value,
|
||||
value_oob_fill_value,
|
||||
abort_when_nan_found,
|
||||
)
|
||||
if kernel_output_idx is not output_idx:
|
||||
output_idx.copy_(kernel_output_idx)
|
||||
return values, output_idx
|
||||
|
||||
|
||||
deepselect_topk = topk
|
||||
Reference in New Issue
Block a user