Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: BBuf <1182563586@qq.com>
179 lines
5.2 KiB
Plaintext
179 lines
5.2 KiB
Plaintext
/// \file vec.cuh
|
|
/// \brief Aligned vector types for coalesced global memory access.
|
|
///
|
|
/// `AlignedVector<T, N>` wraps `N` elements of type `T` in a naturally
|
|
/// aligned struct so that the compiler emits wide (vectorized) load/store
|
|
/// instructions (e.g. `LDG.128`). The maximum supported vector width is
|
|
/// 256 bits (32 bytes), matching CUDA's widest vector load.
|
|
|
|
#pragma once
|
|
#include <sgl_kernel/type.cuh>
|
|
#include <sgl_kernel/utils.cuh>
|
|
|
|
#include <cstddef>
|
|
#include <cstdint>
|
|
|
|
namespace sglang {
|
|
|
|
namespace device {
|
|
|
|
namespace details {
|
|
|
|
/// \brief Maps byte-width to the corresponding unsigned integer type.
|
|
template <std::size_t N>
|
|
struct uint_trait {};
|
|
|
|
template <>
|
|
struct uint_trait<1> {
|
|
using type = uint8_t;
|
|
};
|
|
|
|
template <>
|
|
struct uint_trait<2> {
|
|
using type = uint16_t;
|
|
};
|
|
|
|
template <>
|
|
struct uint_trait<4> {
|
|
using type = uint32_t;
|
|
};
|
|
|
|
template <>
|
|
struct uint_trait<8> {
|
|
using type = uint64_t;
|
|
};
|
|
|
|
/// \brief Alias: maps `sizeof(T)` to matching unsigned int type.
|
|
template <typename T>
|
|
using sized_int = typename uint_trait<sizeof(T)>::type;
|
|
|
|
} // namespace details
|
|
|
|
/// \brief Raw aligned storage for `N` elements of type `T`.
|
|
template <typename T, std::size_t N>
|
|
struct alignas(sizeof(T) * N) AlignedStorage {
|
|
T data[N];
|
|
};
|
|
|
|
/**
|
|
* \brief Aligned vector for vectorized memory access on GPU.
|
|
*
|
|
* Stores `N` elements of type `T` with natural alignment so that a single
|
|
* `load`/`store` call compiles to a wide memory transaction.
|
|
*
|
|
* \tparam T Element type (e.g. `fp16_t`, `bf16_t`, `float`).
|
|
* \tparam N Number of elements. Must be a power of two and
|
|
* `sizeof(T) * N <= 32` (256 bits).
|
|
*
|
|
* Example:
|
|
* \code
|
|
* AlignedVector<fp16_t, 8> vec; // 16 bytes, 128-bit aligned
|
|
* vec.load(input_ptr, tid); // vectorized load
|
|
* vec[0] = vec[0] + 1;
|
|
* vec.store(output_ptr, tid); // vectorized store
|
|
* \endcode
|
|
*/
|
|
template <typename T, std::size_t N>
|
|
struct AlignedVector {
|
|
private:
|
|
static_assert(
|
|
(N > 0 && (N & (N - 1)) == 0) && sizeof(T) * N <= kMaxVecBytes,
|
|
"CUDA vector size exceeds arch limit: max 16 bytes on pre-Blackwell/AMD, "
|
|
"32 bytes on Blackwell or greater");
|
|
using element_t = typename details::sized_int<T>;
|
|
using storage_t = AlignedStorage<element_t, N>;
|
|
|
|
public:
|
|
/// \brief Vectorized load from `ptr` at the given element `offset`.
|
|
SGL_DEVICE void load(const void* ptr, int64_t offset = 0) {
|
|
m_storage = reinterpret_cast<const storage_t*>(ptr)[offset];
|
|
}
|
|
/// \brief Vectorized store to `ptr` at the given element `offset`.
|
|
SGL_DEVICE void store(void* ptr, int64_t offset = 0) const {
|
|
reinterpret_cast<storage_t*>(ptr)[offset] = m_storage;
|
|
}
|
|
/// \brief Fill all N elements with the same `value`.
|
|
SGL_DEVICE void fill(T value) {
|
|
const auto store_value = *reinterpret_cast<element_t*>(&value);
|
|
#pragma unroll
|
|
for (std::size_t i = 0; i < N; ++i) {
|
|
m_storage.data[i] = store_value;
|
|
}
|
|
}
|
|
|
|
SGL_DEVICE auto operator[](std::size_t idx) -> T& {
|
|
return reinterpret_cast<T*>(&m_storage)[idx];
|
|
}
|
|
SGL_DEVICE auto operator[](std::size_t idx) const -> T {
|
|
return reinterpret_cast<const T*>(&m_storage)[idx];
|
|
}
|
|
SGL_DEVICE auto data() -> T* {
|
|
return reinterpret_cast<T*>(&m_storage);
|
|
}
|
|
SGL_DEVICE auto data() const -> const T* {
|
|
return reinterpret_cast<const T*>(&m_storage);
|
|
}
|
|
|
|
private:
|
|
storage_t m_storage;
|
|
};
|
|
|
|
/// \brief Maximum vector width for coalesced memory access on GPU.
|
|
struct LoadStoreBytes {
|
|
enum type : int64_t {
|
|
MAX_GMEM = device::kMaxVecBytes, // architecture-dependent
|
|
MAX_SMEM = 16, // smem only support 16B op
|
|
MAX_PORTABLE = 16, // general across CUDA/HIP
|
|
MIN_COALALESCED = 4, // minimum for coalesced access
|
|
// some common vector widths for load/store
|
|
RAW_1B = 1,
|
|
RAW_2B = 2,
|
|
RAW_4B = 4,
|
|
RAW_8B = 8,
|
|
RAW_16B = 16,
|
|
RAW_32B = 32,
|
|
}; // namespace device
|
|
};
|
|
|
|
/// Sum `M` vectors element-wise into one, accumulating in fp32 regardless of
|
|
/// the packed element type. Used by every collective that reduces peer
|
|
/// contributions in registers.
|
|
template <bool kFP32Acc = true, typename T2, size_t N, size_t M>
|
|
SGL_DEVICE auto reduce_vec(device::AlignedVector<T2, N> (&vec)[M]) -> device::AlignedVector<T2, N> {
|
|
static_assert(DTypeTrait<T2>::kVecSize == 2, "reduce_vec only supports 2-element vectors for now");
|
|
static_assert(M > 0, "reduce_vec requires at least one vector to reduce");
|
|
if constexpr (kFP32Acc) {
|
|
fp32x2_t acc[N];
|
|
#pragma unroll
|
|
for (size_t i = 0; i < M; ++i) {
|
|
#pragma unroll
|
|
for (size_t j = 0; j < N; ++j) {
|
|
const auto [x, y] = cast<fp32x2_t>(vec[i][j]);
|
|
acc[j].x = i == 0 ? x : acc[j].x + x;
|
|
acc[j].y = i == 0 ? y : acc[j].y + y;
|
|
}
|
|
}
|
|
device::AlignedVector<T2, N> out_vec;
|
|
#pragma unroll
|
|
for (size_t j = 0; j < N; ++j) {
|
|
out_vec[j] = cast<T2>(acc[j]);
|
|
}
|
|
return out_vec;
|
|
} else {
|
|
using SumOp = ReductionTrait<ReductionOp::SUM, T2>;
|
|
device::AlignedVector<T2, N> out_vec = vec[0];
|
|
#pragma unroll
|
|
for (size_t i = 1; i < M; ++i) {
|
|
#pragma unroll
|
|
for (size_t j = 0; j < N; ++j) {
|
|
out_vec[j] = SumOp::apply(out_vec[j], vec[i][j]);
|
|
}
|
|
}
|
|
return out_vec;
|
|
}
|
|
}
|
|
|
|
} // namespace device
|
|
|
|
} // namespace sglang
|