[JIT] Clean up kernel headers: move runtime helpers, add once.h (#41147)
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
b50fdeb0ef
commit
4f5fcbfef5
@@ -29,12 +29,12 @@ using F2S_T = int64_t;
|
||||
using IDX_T = int64_t;
|
||||
|
||||
/// NOTE: for the internal use, we pack the ragged and batch id, since both not exceed 65536
|
||||
SGL_DEVICE __host__ PlanW pack_w(uint32_t ragged_id, uint32_t batch_id, int32_t seq_len) {
|
||||
SGL_DEVICE_HOST PlanW pack_w(uint32_t ragged_id, uint32_t batch_id, int32_t seq_len) {
|
||||
return {static_cast<uint32_t>(ragged_id | batch_id << 16), seq_len};
|
||||
}
|
||||
|
||||
/// NOTE: for the internal use, we pack the ragged and batch id, since both not exceed 65536
|
||||
SGL_DEVICE uint2 unpack_w(PlanW plan) {
|
||||
SGL_DEVICE_HOST uint2 unpack_w(PlanW plan) {
|
||||
return {static_cast<uint16_t>(plan.ragged_id), static_cast<uint16_t>(plan.ragged_id >> 16)};
|
||||
}
|
||||
|
||||
|
||||
@@ -85,12 +85,6 @@ struct TopKBF16Params {
|
||||
uint32_t page_bits;
|
||||
};
|
||||
|
||||
SGL_DEVICE uint32_t get_ptx_lane_id() {
|
||||
uint32_t lane_id;
|
||||
asm volatile("mov.u32 %0, %%laneid;" : "=r"(lane_id));
|
||||
return lane_id;
|
||||
}
|
||||
|
||||
/// \brief Exclusive suffix scan: lane `L` gets the sum over lanes `> L`.
|
||||
SGL_DEVICE uint32_t warp_exclusive_suffix_sum(uint32_t x, uint32_t lane_id) {
|
||||
uint32_t inc = x;
|
||||
@@ -183,7 +177,7 @@ __global__ __launch_bounds__(TopKBF16Config::kBlockSize, TopKBF16Config::kOccupa
|
||||
|
||||
const auto bx = blockIdx.x;
|
||||
const auto tx = threadIdx.x;
|
||||
const auto lane_id = get_ptx_lane_id();
|
||||
const auto lane_id = warp::get_lane_id();
|
||||
const auto warp_id = tx / kWarpThreads;
|
||||
const auto topk = params.topk;
|
||||
// a selected index i maps through this row's table to slot
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
#include <sgl_kernel/tensor.h>
|
||||
#include <sgl_kernel/utils.h>
|
||||
|
||||
#include <sgl_kernel/runtime.cuh>
|
||||
#include <sgl_kernel/utils.cuh>
|
||||
|
||||
#include <dlpack/dlpack.h>
|
||||
@@ -14,8 +15,8 @@ namespace sglang {
|
||||
// `topk` is a *runtime* value (<= kMaxTopK), so one module serves every k. It
|
||||
// used to be baked in via -DSGL_TOPK, which built a separate module per k --
|
||||
// and because `kTopK` came from a macro rather than a template parameter, both
|
||||
// modules exported identically mangled symbols. The function-local static in
|
||||
// setup_kernel_smem_once() is emitted as STB_GNU_UNIQUE, which the loader
|
||||
// modules exported identically mangled symbols. The function-local static that
|
||||
// guarded the smem opt-in is emitted as STB_GNU_UNIQUE, which the loader
|
||||
// merges across every loaded object, so whichever module was used second
|
||||
// skipped its cudaFuncSetAttribute opt-in and then failed to launch with 64 KB
|
||||
// of dynamic shared memory ("invalid argument").
|
||||
@@ -270,16 +271,6 @@ __global__ void topk_transform_kernel(const __grid_constant__ TopKParams params)
|
||||
device::PDLTriggerSecondary<kUsePDL>();
|
||||
}
|
||||
|
||||
template <auto* f, size_t kMaxDynamicSMEM>
|
||||
void setup_kernel_smem_once(host::DebugInfo where = {}) {
|
||||
[[maybe_unused]]
|
||||
static const auto result = [] {
|
||||
const auto fptr = std::bit_cast<const void*>(f);
|
||||
return ::cudaFuncSetAttribute(fptr, ::cudaFuncAttributeMaxDynamicSharedMemorySize, kMaxDynamicSMEM);
|
||||
}();
|
||||
host::RuntimeDeviceCheck(result, where);
|
||||
}
|
||||
|
||||
template <bool kUsePDL>
|
||||
struct TopKKernel {
|
||||
static constexpr auto kernel = topk_transform_kernel<kUsePDL>;
|
||||
@@ -344,7 +335,7 @@ struct TopKKernel {
|
||||
.topk = topk,
|
||||
};
|
||||
constexpr auto kSMEM_ = kSMEM + sizeof(int32_t); // align up a little
|
||||
setup_kernel_smem_once<kernel, kSMEM_>();
|
||||
runtime::set_max_dynamic_smem_per_device<kernel>(device.unwrap(), kSMEM_);
|
||||
LaunchKernel(batch_size, kTopKBlockSize, device.unwrap(), kSMEM_).enable_pdl(kUsePDL)(kernel, params);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
#include <sgl_kernel/tensor.h>
|
||||
#include <sgl_kernel/utils.h>
|
||||
|
||||
#include <sgl_kernel/runtime.cuh>
|
||||
#include <sgl_kernel/utils.cuh>
|
||||
|
||||
#include <sgl_kernel/deepseek_v4/topk_impl.cuh>
|
||||
@@ -1103,12 +1104,10 @@ struct TopKKernel {
|
||||
constexpr uint32_t kClusterSize = 16;
|
||||
// Widths above 8 are non-portable; the launch is rejected without this.
|
||||
const auto kernel = topk_small_batch_cluster_kernel<kUsePDL, kMode, kClusterSize, 1>;
|
||||
[[maybe_unused]]
|
||||
static const bool _ = [&kernel] {
|
||||
const auto kernel_ptr = reinterpret_cast<const void*>(kernel);
|
||||
const auto kernel_ptr = reinterpret_cast<const void*>(kernel);
|
||||
runtime::init_per_device(device, [kernel_ptr] {
|
||||
CHECK_CUDA(::cudaFuncSetAttribute(kernel_ptr, ::cudaFuncAttributeNonPortableClusterSizeAllowed, 1));
|
||||
return true;
|
||||
}();
|
||||
});
|
||||
return LaunchKernel({batch_size, kClusterSize}, kBlockSize, device)
|
||||
.config({.use_pdl = kUsePDL, .cluster_dim = dim3{1, kClusterSize}})
|
||||
.launch(kernel, params);
|
||||
|
||||
@@ -22,6 +22,7 @@
|
||||
#include <sgl_kernel/utils.h>
|
||||
|
||||
#include <sgl_kernel/mbarrier.cuh>
|
||||
#include <sgl_kernel/runtime.cuh>
|
||||
#include <sgl_kernel/type.cuh>
|
||||
#include <sgl_kernel/utils.cuh>
|
||||
#include <sgl_kernel/warp.cuh>
|
||||
@@ -779,11 +780,10 @@ inline void wo_a_fused_run(
|
||||
const auto launch = [&](const auto* pos) {
|
||||
using PosT = std::decay_t<decltype(*pos)>;
|
||||
constexpr auto kernel = wo_a_fused_kernel<Trait, PosT>;
|
||||
[[maybe_unused]] static const auto _ = [] {
|
||||
CHECK_CUDA(cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, sizeof(Smem)));
|
||||
CHECK_CUDA(cudaFuncSetAttribute(kernel, cudaFuncAttributePreferredSharedMemoryCarveout, 100));
|
||||
return 0;
|
||||
}();
|
||||
runtime::init_per_device(device, [=] {
|
||||
runtime::set_max_dynamic_smem(kernel, sizeof(Smem));
|
||||
runtime::set_smem_carveout(kernel, cudaSharedmemCarveoutMaxShared);
|
||||
});
|
||||
LaunchKernel(kClusters * kSplitK, kThreads, device, sizeof(Smem))
|
||||
.config({.use_pdl = true, .cluster_dim = dim3{kSplitK, 1, 1}})(
|
||||
kernel,
|
||||
|
||||
@@ -3,12 +3,12 @@
|
||||
#include <sgl_kernel/tensor.h>
|
||||
#include <sgl_kernel/utils.h>
|
||||
|
||||
#include <sgl_kernel/runtime.cuh>
|
||||
#include <sgl_kernel/utils.cuh>
|
||||
|
||||
#include <dlpack/dlpack.h>
|
||||
#include <tvm/ffi/container/tensor.h>
|
||||
|
||||
#include <bit>
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#ifndef USE_ROCM
|
||||
@@ -354,20 +354,6 @@ __global__ __launch_bounds__(kThreadsPerBlock) void kpool_topk_transform_kernel(
|
||||
}
|
||||
}
|
||||
|
||||
template <auto* f, std::size_t kMaxDynamicSMEM>
|
||||
void setup_kernel_smem_once(host::DebugInfo where = {}) {
|
||||
[[maybe_unused]]
|
||||
static const auto result = [] {
|
||||
const auto fptr = std::bit_cast<const void*>(f);
|
||||
#if defined(__HIP_PLATFORM_AMD__)
|
||||
return ::hipFuncSetAttribute(fptr, ::hipFuncAttributeMaxDynamicSharedMemorySize, kMaxDynamicSMEM);
|
||||
#else
|
||||
return ::cudaFuncSetAttribute(fptr, ::cudaFuncAttributeMaxDynamicSharedMemorySize, kMaxDynamicSMEM);
|
||||
#endif
|
||||
}();
|
||||
host::RuntimeDeviceCheck(result, where);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
const T* optional_data_ptr(const tvm::ffi::Optional<tvm::ffi::TensorView>& opt) {
|
||||
if (!opt.has_value()) {
|
||||
@@ -463,7 +449,7 @@ struct KpoolTopKTransformKernel {
|
||||
.input_stride = static_cast<int64_t>(S.unwrap()),
|
||||
};
|
||||
|
||||
setup_kernel_smem_once<kernel, kSmem>();
|
||||
runtime::set_max_dynamic_smem_per_device<kernel>(device.unwrap().device_id, kSmem);
|
||||
LaunchKernel(batch_size, kThreadsPerBlock, device.unwrap(), kSmem)(
|
||||
kernel,
|
||||
params,
|
||||
|
||||
@@ -48,6 +48,7 @@ limitations under the License.
|
||||
#include <cutlass/util/packed_stride.hpp>
|
||||
#include <dlpack/dlpack.h>
|
||||
#include <tvm/ffi/container/tensor.h>
|
||||
#include <tvm/ffi/extra/cuda/device_guard.h>
|
||||
|
||||
#include "cutlass_extensions/epilogue/epilogue_per_row_per_col_scale.h"
|
||||
#include "cutlass_extensions/gemm/gemm_universal_base_compat.h"
|
||||
|
||||
@@ -21,6 +21,7 @@
|
||||
#include <sgl_kernel/tensor.h>
|
||||
#include <sgl_kernel/utils.h>
|
||||
|
||||
#include <sgl_kernel/runtime.cuh>
|
||||
#include <sgl_kernel/utils.cuh>
|
||||
|
||||
#include <tvm/ffi/container/tensor.h>
|
||||
@@ -593,9 +594,9 @@ void invokeFusedAGemm(T* output, T const* mat_a, T const* mat_b, int num_tokens,
|
||||
dim3 grid(cta_m_cnt, cta_n_cnt, 1);
|
||||
dim3 block_size(256);
|
||||
|
||||
auto kernel = fused_a_gemm_kernel<batch_size, gemm_m, gemm_k, tile_m, tile_n, tile_k, stage_cnt>;
|
||||
if (smem_bytes >= (48 * 1024)) {
|
||||
host::RuntimeDeviceCheck(cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, smem_bytes));
|
||||
constexpr auto kernel = fused_a_gemm_kernel<batch_size, gemm_m, gemm_k, tile_m, tile_n, tile_k, stage_cnt>;
|
||||
if constexpr (smem_bytes >= (48 * 1024)) {
|
||||
host::runtime::set_max_dynamic_smem_per_device<kernel>(device.device_id, smem_bytes);
|
||||
}
|
||||
host::LaunchKernel(grid, block_size, device, smem_bytes).enable_pdl(kUsePDL)(kernel, output, mat_a, mat_b, gemm_n);
|
||||
}
|
||||
|
||||
@@ -28,6 +28,7 @@
|
||||
#include <sgl_kernel/utils.h>
|
||||
|
||||
#include <sgl_kernel/mbarrier.cuh>
|
||||
#include <sgl_kernel/runtime.cuh>
|
||||
#include <sgl_kernel/utils.cuh>
|
||||
#include <sgl_kernel/warp.cuh>
|
||||
|
||||
@@ -130,7 +131,7 @@ inline constexpr uint32_t kHicacheTmaMaxMapRowBytes = 256 * 8;
|
||||
|
||||
// Rows per chunk: largest power of two that fits the stage, so chunks never
|
||||
// straddle a (power-of-two) page and a page run stays one bulk op.
|
||||
__host__ __device__ constexpr uint32_t hicache_tma_rows_per_chunk(uint32_t stage_bytes, uint32_t row_bytes) {
|
||||
SGL_DEVICE_HOST constexpr uint32_t hicache_tma_rows_per_chunk(uint32_t stage_bytes, uint32_t row_bytes) {
|
||||
uint32_t rows = 1;
|
||||
while (rows * 2 <= stage_bytes / row_bytes && rows * 2 <= kHicacheTmaMaxRows)
|
||||
rows *= 2;
|
||||
@@ -438,16 +439,12 @@ struct HiCacheTmaKernel {
|
||||
(params.has_v ? 2u : 1u) * params.num_layers * div_ceil(params.length, rows_per_chunk(params.row_bytes));
|
||||
constexpr std::size_t kSmemBytes = sizeof(Smem);
|
||||
|
||||
static const bool attr_set = [] {
|
||||
for (auto fn : {kernel<int32_t>, kernel<int64_t>}) {
|
||||
RuntimeDeviceCheck(
|
||||
cudaFuncSetAttribute(fn, cudaFuncAttributeMaxDynamicSharedMemorySize, static_cast<int>(kSmemBytes)));
|
||||
}
|
||||
return true;
|
||||
}();
|
||||
(void)attr_set;
|
||||
LaunchKernel(std::min(chunks, kBlockQuota), kThreads, device, kSmemBytes)(
|
||||
use_int32 ? kernel<int32_t> : kernel<int64_t>, params);
|
||||
runtime::init_per_device(device, [=] {
|
||||
runtime::set_max_dynamic_smem(kernel<int32_t>, kSmemBytes);
|
||||
runtime::set_max_dynamic_smem(kernel<int64_t>, kSmemBytes);
|
||||
});
|
||||
const auto fn = use_int32 ? kernel<int32_t> : kernel<int64_t>;
|
||||
LaunchKernel(std::min(chunks, kBlockQuota), kThreads, device, kSmemBytes)(fn, params);
|
||||
}
|
||||
|
||||
// Cache operand viewed as [-1, D] rows; binds row dim, stride and dtype.
|
||||
|
||||
@@ -779,7 +779,10 @@ void load_cache_to_device_buffer(
|
||||
constexpr size_t smem_bytes = SmemLayout<NUM_TOP_K_TOKENS, HOT_BUFFER_SIZE>::BYTES;
|
||||
#ifndef USE_ROCM
|
||||
if constexpr (smem_bytes > 48u * 1024u) {
|
||||
cudaFuncSetAttribute(kernel_fn, cudaFuncAttributeMaxDynamicSharedMemorySize, smem_bytes);
|
||||
/// NOTE: for each launch template, the `kernel_fn` must be unique
|
||||
runtime::init_per_device(kernel_device, [=] { //
|
||||
runtime::set_max_dynamic_smem(kernel_fn, smem_bytes);
|
||||
});
|
||||
}
|
||||
#endif
|
||||
LaunchKernel(bs, BLOCK_SIZE, device, smem_bytes)(
|
||||
|
||||
@@ -18,6 +18,7 @@
|
||||
|
||||
#include <sgl_kernel/tensor.h>
|
||||
|
||||
#include <sgl_kernel/runtime.cuh>
|
||||
#include <sgl_kernel/utils.cuh>
|
||||
|
||||
#include <sgl_kernel/speculative/sampling.cuh>
|
||||
@@ -78,7 +79,7 @@ void tree_speculative_sampling_target_only(
|
||||
|
||||
constexpr uint32_t block_threads = 1024;
|
||||
constexpr size_t smem_size = sizeof(SamplingTempStorage<block_threads, SCAN_ALGO, REDUCE_ALGO>);
|
||||
auto kernel = speculative_sampling::TreeSpeculativeSamplingTargetOnly<
|
||||
constexpr auto kernel = speculative_sampling::TreeSpeculativeSamplingTargetOnly<
|
||||
block_threads,
|
||||
SCAN_ALGO,
|
||||
REDUCE_ALGO,
|
||||
@@ -87,7 +88,7 @@ void tree_speculative_sampling_target_only(
|
||||
float,
|
||||
int32_t,
|
||||
int64_t>;
|
||||
CHECK_CUDA(cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, smem_size));
|
||||
runtime::set_max_dynamic_smem_per_device<kernel>(device.unwrap().device_id, smem_size);
|
||||
LaunchKernel(static_cast<uint32_t>(batch_size.unwrap()), block_threads, device.unwrap(), smem_size)(
|
||||
kernel,
|
||||
static_cast<int32_t*>(predicts.data_ptr()),
|
||||
|
||||
@@ -1,40 +0,0 @@
|
||||
#pragma once
|
||||
|
||||
#include <concepts>
|
||||
#include <mutex>
|
||||
#include <type_traits>
|
||||
#include <unordered_map>
|
||||
|
||||
namespace sglang::host {
|
||||
|
||||
/**
|
||||
* \brief Allocate only once for a given function.
|
||||
* \tparam kThreadSafe Whether to make the allocation thread-safe.
|
||||
* \tparam Salt A salt type to avoid cache collision.
|
||||
* \param key The key to identify the allocation. It should be unique for each allocation.
|
||||
* \param callback The callback function to perform the allocation. It should return the allocated value.
|
||||
* \note The `Fn` type must be unique. It's typically a lambda type that's evaluated only once.
|
||||
* Otherwise, different call-sites may hit the same cache entry.
|
||||
* In case where `Fn` is not unique (e.g. std::function), make `Salt` unique to avoid cache collision.
|
||||
*/
|
||||
template <bool kThreadSafe = true, typename Salt = void, typename Key, std::invocable Fn>
|
||||
inline auto allocate_once(Key&& key, Fn&& callback) -> std::decay_t<std::invoke_result_t<Fn>>& {
|
||||
using Value = std::decay_t<std::invoke_result_t<Fn>>;
|
||||
static std::unordered_map<std::decay_t<Key>, Value> s_map;
|
||||
const auto alloc = [&]() -> Value& {
|
||||
const auto iter = s_map.find(key);
|
||||
if (iter != s_map.end()) return iter->second;
|
||||
// Evaluate the callback before inserting, so a throwing callback leaves no empty entry behind.
|
||||
auto value = std::forward<Fn>(callback)();
|
||||
return s_map.emplace(std::forward<Key>(key), std::move(value)).first->second;
|
||||
};
|
||||
if constexpr (kThreadSafe) {
|
||||
static std::mutex s_mutex;
|
||||
const auto lock = std::lock_guard{s_mutex};
|
||||
return alloc();
|
||||
} else {
|
||||
return alloc();
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace sglang::host
|
||||
@@ -0,0 +1,54 @@
|
||||
#pragma once
|
||||
|
||||
#include <sgl_kernel/utils.h>
|
||||
|
||||
#include <cstdint>
|
||||
#include <cstdlib>
|
||||
#include <iostream>
|
||||
#include <sstream>
|
||||
#include <string_view>
|
||||
|
||||
namespace sglang::host {
|
||||
|
||||
enum class LogLevel : uint8_t {
|
||||
DEBUG_ = 0,
|
||||
INFO_ = 1,
|
||||
WARNING_ = 2,
|
||||
ERROR_ = 3,
|
||||
DEFAULT = WARNING_,
|
||||
};
|
||||
|
||||
inline const LogLevel kLogLevel = [] {
|
||||
const auto env = std::getenv("SGLANG_JIT_LOG_LEVEL");
|
||||
if (!env) return LogLevel::DEFAULT;
|
||||
const auto str = std::string_view{env};
|
||||
if (str == "DEBUG" || str == "debug") return LogLevel::DEBUG_;
|
||||
if (str == "INFO" || str == "info") return LogLevel::INFO_;
|
||||
if (str == "WARNING" || str == "warning") return LogLevel::WARNING_;
|
||||
if (str == "ERROR" || str == "error") return LogLevel::ERROR_;
|
||||
std::clog << "[WARNING] unrecognized SGLANG_JIT_LOG_LEVEL value '" << str << "'; using default log level."
|
||||
<< std::endl;
|
||||
return LogLevel::DEFAULT;
|
||||
}();
|
||||
|
||||
struct Logger {
|
||||
std::ostringstream stream;
|
||||
explicit Logger(LogLevel level, DebugInfo location = {}) {
|
||||
static const std::string_view kPrefix[] = {"[DEBUG]", "[INFO]", "[WARNING]", "[ERROR]"};
|
||||
stream << kPrefix[static_cast<size_t>(level)] << " [" << location.file_name() << ":" << location.line() << "] ";
|
||||
}
|
||||
~Logger() {
|
||||
std::clog << std::move(stream).str() << std::endl;
|
||||
}
|
||||
};
|
||||
|
||||
#define SGL_LOG(LEVEL) \
|
||||
if (const auto _log_level = ::sglang::host::LogLevel::LEVEL; _log_level < ::sglang::host::kLogLevel) { \
|
||||
} else \
|
||||
::sglang::host::Logger(_log_level).stream
|
||||
|
||||
#define SGL_LOG_DEBUG SGL_LOG(DEBUG_)
|
||||
#define SGL_LOG_INFO SGL_LOG(INFO_)
|
||||
#define SGL_LOG_WARNING SGL_LOG(WARNING_)
|
||||
|
||||
} // namespace sglang::host
|
||||
@@ -0,0 +1,147 @@
|
||||
/// \file once.h
|
||||
/// \brief Host-side "run it only once" helpers.
|
||||
///
|
||||
/// C++ spells this `std::call_once` plus a caller-provided `std::once_flag`. The helpers here
|
||||
/// hide the flag inside the function, keyed off the uniqueness of the callable's type, and hand
|
||||
/// back the value the callable produced. They come in two forms: one value per call-site, or
|
||||
/// one value per (call-site, key).
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <concepts>
|
||||
#include <cstddef>
|
||||
#include <functional>
|
||||
#include <mutex>
|
||||
#include <tuple>
|
||||
#include <type_traits>
|
||||
#include <unordered_map>
|
||||
#include <unordered_set>
|
||||
#include <utility>
|
||||
|
||||
namespace sglang::host {
|
||||
|
||||
namespace details {
|
||||
|
||||
/// \brief Fold any number of sub-hashes into one; the mixing step is boost's.
|
||||
/// Defined for zero parts too, so an empty tuple key hashes without a special case.
|
||||
inline std::size_t hash_combine(std::same_as<std::size_t> auto... parts) {
|
||||
std::size_t seed = 0;
|
||||
((seed ^= parts + 0x9e3779b97f4a7c15ULL + (seed << 6) + (seed >> 2)), ...);
|
||||
return seed;
|
||||
}
|
||||
|
||||
/// \brief Hash for keys `std::hash` does not cover: pairs and tuples, recursively.
|
||||
struct GenericHash {
|
||||
template <typename T>
|
||||
std::size_t operator()(const T& value) const {
|
||||
return std::hash<T>{}(value);
|
||||
}
|
||||
template <typename T, typename U>
|
||||
std::size_t operator()(const std::pair<T, U>& value) const {
|
||||
return hash_combine((*this)(value.first), (*this)(value.second));
|
||||
}
|
||||
template <typename... Args>
|
||||
std::size_t operator()(const std::tuple<Args...>& value) const {
|
||||
return std::apply([this](const auto&... args) { return hash_combine((*this)(args)...); }, value);
|
||||
}
|
||||
};
|
||||
|
||||
struct DummyMutex {
|
||||
void lock() {}
|
||||
void unlock() {}
|
||||
};
|
||||
|
||||
} // namespace details
|
||||
|
||||
template <typename Key, typename Value, bool kThreadSafe = true>
|
||||
struct CacheMap {
|
||||
public:
|
||||
CacheMap() = default;
|
||||
CacheMap(const CacheMap&) = delete;
|
||||
CacheMap& operator=(const CacheMap&) = delete;
|
||||
|
||||
template <typename Fn, typename KeyArg>
|
||||
auto get_cached(KeyArg&& key, Fn&& factory) -> Value& {
|
||||
const auto lock = std::lock_guard{m_mutex};
|
||||
if (const auto iter = m_map.find(key); iter != m_map.end()) return iter->second;
|
||||
// Evaluate the factory before inserting, so a throwing factory leaves no entry behind.
|
||||
auto value = std::forward<Fn>(factory)();
|
||||
return m_map.try_emplace(std::forward<KeyArg>(key), std::move(value)).first->second;
|
||||
}
|
||||
|
||||
private:
|
||||
[[no_unique_address]]
|
||||
std::conditional_t<kThreadSafe, std::mutex, details::DummyMutex> m_mutex;
|
||||
std::unordered_map<Key, Value, details::GenericHash> m_map;
|
||||
};
|
||||
|
||||
template <typename Key, bool kThreadSafe>
|
||||
struct CacheMap<Key, void, kThreadSafe> {
|
||||
public:
|
||||
CacheMap() = default;
|
||||
CacheMap(const CacheMap&) = delete;
|
||||
CacheMap& operator=(const CacheMap&) = delete;
|
||||
|
||||
template <typename Fn, typename KeyArg>
|
||||
void get_cached(KeyArg&& key, Fn&& factory) {
|
||||
const auto lock = std::lock_guard{m_mutex};
|
||||
if (m_set.contains(key)) return;
|
||||
// Run the factory before inserting, so a throwing factory is retried on the next call.
|
||||
std::forward<Fn>(factory)();
|
||||
m_set.emplace(std::forward<KeyArg>(key));
|
||||
}
|
||||
|
||||
private:
|
||||
[[no_unique_address]]
|
||||
std::conditional_t<kThreadSafe, std::mutex, details::DummyMutex> m_mutex;
|
||||
std::unordered_set<Key, details::GenericHash> m_set;
|
||||
};
|
||||
|
||||
template <typename Key, bool kThreadSafe = true>
|
||||
using CacheSet = CacheMap<Key, void, kThreadSafe>;
|
||||
|
||||
/**
|
||||
* \brief A keyed `std::call_once` that also hands back the value.
|
||||
* Runs `factory` at most once per (call-site, key) and returns a stable reference to its result.
|
||||
* \tparam Tag A tag type to avoid cache collision.
|
||||
* \tparam kThreadSafe Whether to make the initialization thread-safe. Turn it off only where the
|
||||
* call-site is provably single-threaded: the lock is taken on every call, cache hits included.
|
||||
* \param key The key to identify the value. Each distinct key is initialized independently.
|
||||
* \param factory The function to produce the value on first use. It should return the value.
|
||||
* \note The `Fn` type must be unique. It's typically a lambda type that's evaluated only once.
|
||||
* Otherwise, different call-sites may hit the same cache entry.
|
||||
* In case where `Fn` is not unique (e.g. std::function), make `Tag` unique to avoid cache collision.
|
||||
* \note `factory` runs with the lock held, so it must not re-enter this same instantiation.
|
||||
*/
|
||||
template <typename Tag = void, bool kThreadSafe = true, typename Key, std::invocable Fn>
|
||||
inline auto init_once(Key&& key, Fn&& factory) -> decltype(auto) {
|
||||
static CacheMap<std::decay_t<Key>, std::decay_t<std::invoke_result_t<Fn>>, kThreadSafe> s_cache;
|
||||
return s_cache.get_cached(std::forward<Key>(key), std::forward<Fn>(factory));
|
||||
}
|
||||
|
||||
/**
|
||||
* \brief `std::call_once` without the caller-provided `once_flag`, also handing back the value.
|
||||
* Runs `factory` at most once per call-site and returns a stable reference to its result.
|
||||
* \tparam Tag A tag type to avoid cache collision.
|
||||
* \param factory The function to produce the value on first use. It should return the value.
|
||||
* \note The `Fn` type must be unique, for the same reason as the keyed overload above: the static
|
||||
* lives in the instantiation, so a non-unique `Fn` (e.g. std::function) makes two call-sites share
|
||||
* one value. Make `Tag` unique in that case.
|
||||
* \note Always thread-safe, and deliberately has no `kThreadSafe` knob: C++ already guarantees that
|
||||
* concurrent callers block until the first one finishes, and the post-initialization fast path is
|
||||
* just a guard check, so there is no lock worth skipping.
|
||||
* \note If `factory` throws, the value stays uninitialized and the next call retries, matching the
|
||||
* keyed overload. `factory` must not re-enter this same instantiation; that is undefined behavior.
|
||||
*/
|
||||
template <typename Tag = void, std::invocable Fn>
|
||||
inline auto init_once(Fn&& factory) -> decltype(auto) {
|
||||
if constexpr (std::is_same_v<std::invoke_result_t<Fn>, void>) {
|
||||
static std::once_flag s_once;
|
||||
return std::call_once(s_once, std::forward<Fn>(factory));
|
||||
} else {
|
||||
static auto result = std::forward<Fn>(factory)();
|
||||
return (result); // parenthesized: `decltype(auto)` then deduces a reference, not a copy
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace sglang::host
|
||||
@@ -6,13 +6,19 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <sgl_kernel/utils.h>
|
||||
|
||||
#include <sgl_kernel/utils.cuh>
|
||||
|
||||
#include <dlpack/dlpack.h>
|
||||
#include <tvm/ffi/container/tensor.h>
|
||||
|
||||
#include <atomic>
|
||||
#include <concepts>
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <utility>
|
||||
|
||||
#ifndef USE_ROCM
|
||||
#include <cuda_runtime.h>
|
||||
#else
|
||||
@@ -59,21 +65,22 @@ namespace sglang {
|
||||
|
||||
namespace host::runtime {
|
||||
|
||||
inline void* get_device_accessible_ptr(const tvm::ffi::TensorView& tensor) {
|
||||
void* ptr = tensor.data_ptr();
|
||||
const auto tensor_device_type = tensor.device().device_type;
|
||||
if (tensor_device_type != kDLCPU && tensor_device_type != kDLGPUHost) {
|
||||
return ptr;
|
||||
}
|
||||
namespace details {
|
||||
|
||||
void* device_ptr = nullptr;
|
||||
#ifdef USE_ROCM
|
||||
RuntimeDeviceCheck(::hipHostGetDevicePointer(&device_ptr, ptr, 0));
|
||||
#else
|
||||
RuntimeDeviceCheck(::cudaHostGetDevicePointer(&device_ptr, ptr, 0));
|
||||
#endif
|
||||
return device_ptr;
|
||||
}
|
||||
template <auto V>
|
||||
struct Tag {};
|
||||
|
||||
struct MaybeDevice {
|
||||
MaybeDevice(DLDevice device) : device_id(device.device_id) {}
|
||||
MaybeDevice(int32_t device_id_) : device_id(device_id_) {}
|
||||
uint32_t device_id;
|
||||
};
|
||||
|
||||
} // namespace details
|
||||
|
||||
// Generous bound on the device ordinals one process can see; a larger ordinal
|
||||
// is not an error, it just falls through to the driver query uncached.
|
||||
inline constexpr uint32_t kNumStaticMaxDevice = 72;
|
||||
|
||||
/**
|
||||
* \brief Memo for a per-device driver query, keyed by device ordinal.
|
||||
@@ -85,10 +92,6 @@ inline void* get_device_accessible_ptr(const tvm::ffi::TensorView& tensor) {
|
||||
template <typename T, T kDefault>
|
||||
struct DeviceCacheMap {
|
||||
public:
|
||||
// Generous bound on the device ordinals one process can see; a larger ordinal
|
||||
// is not an error, it just falls through to the driver query uncached.
|
||||
static constexpr uint32_t kNumStaticMaxDevice = 72;
|
||||
|
||||
constexpr DeviceCacheMap() {
|
||||
for (uint32_t i = 0; i < kNumStaticMaxDevice; ++i) {
|
||||
m_data[i] = kDefault;
|
||||
@@ -108,13 +111,34 @@ struct DeviceCacheMap {
|
||||
return value;
|
||||
}
|
||||
|
||||
template <typename Fn>
|
||||
T get_cached(int32_t device_, Fn&& fn) {
|
||||
return this->get_cached(device_, true, std::forward<Fn>(fn));
|
||||
}
|
||||
|
||||
private:
|
||||
T m_data[kNumStaticMaxDevice];
|
||||
};
|
||||
|
||||
inline void* get_device_accessible_ptr(const tvm::ffi::TensorView& tensor) {
|
||||
void* ptr = tensor.data_ptr();
|
||||
const auto tensor_device_type = tensor.device().device_type;
|
||||
if (tensor_device_type != kDLCPU && tensor_device_type != kDLGPUHost) {
|
||||
return ptr;
|
||||
}
|
||||
|
||||
void* device_ptr = nullptr;
|
||||
#ifdef USE_ROCM
|
||||
CHECK_CUDA(::hipHostGetDevicePointer(&device_ptr, ptr, 0));
|
||||
#else
|
||||
CHECK_CUDA(::cudaHostGetDevicePointer(&device_ptr, ptr, 0));
|
||||
#endif
|
||||
return device_ptr;
|
||||
}
|
||||
|
||||
// Return the maximum number of active blocks per SM for the given kernel
|
||||
template <typename T>
|
||||
inline auto get_blocks_per_sm(T&& kernel, int32_t block_dim, std::size_t dynamic_smem = 0) -> uint32_t {
|
||||
inline auto get_blocks_per_sm(T kernel, int32_t block_dim, std::size_t dynamic_smem = 0) -> uint32_t {
|
||||
int num_blocks_per_sm = 0;
|
||||
CHECK_CUDA(cudaOccupancyMaxActiveBlocksPerMultiprocessor(&num_blocks_per_sm, kernel, block_dim, dynamic_smem));
|
||||
return static_cast<uint32_t>(num_blocks_per_sm);
|
||||
@@ -164,7 +188,7 @@ inline auto get_runtime_version() -> int {
|
||||
|
||||
// Return the maximum dynamic shared memory per block for the given kernel
|
||||
template <typename T>
|
||||
inline auto get_available_dynamic_smem_per_block(T&& kernel, int num_blocks, int block_size) -> std::size_t {
|
||||
inline auto get_available_dynamic_smem_per_block(T kernel, int num_blocks, int block_size) -> std::size_t {
|
||||
std::size_t smem_size;
|
||||
CHECK_CUDA(cudaOccupancyAvailableDynamicSMemPerBlock(&smem_size, kernel, num_blocks, block_size));
|
||||
return smem_size;
|
||||
@@ -190,6 +214,110 @@ inline auto get_max_smem_per_block(int device_id, bool opt_in = true, bool use_c
|
||||
});
|
||||
}
|
||||
|
||||
struct L1Carveout {
|
||||
int carveout_percent;
|
||||
uint32_t blocks_per_sm;
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
inline void set_smem_carveout(T kernel, int percent) {
|
||||
#ifdef USE_ROCM
|
||||
(void)kernel, (void)percent;
|
||||
#else
|
||||
CHECK_CUDA(::cudaFuncSetAttribute(kernel, cudaFuncAttributePreferredSharedMemoryCarveout, percent));
|
||||
#endif
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
inline auto set_prefer_l1_carveout_with_occupancy(
|
||||
const int lower_bound_percent,
|
||||
const int upper_bound_percent,
|
||||
const uint32_t min_occupancy,
|
||||
T kernel,
|
||||
const uint32_t block_threads,
|
||||
const std::size_t dyn_smem_bytes = 0) -> L1Carveout {
|
||||
#ifdef USE_ROCM
|
||||
(void)lower_bound_percent, (void)upper_bound_percent, (void)min_occupancy;
|
||||
return {-1, get_blocks_per_sm(kernel, block_threads, dyn_smem_bytes)};
|
||||
#else
|
||||
const auto kernel_ptr = reinterpret_cast<const void*>(kernel);
|
||||
// The attribute and the occupancy query both act on the current device.
|
||||
static_assert(cudaSharedmemCarveoutMaxL1 <= cudaSharedmemCarveoutMaxShared);
|
||||
CHECK_HOST(
|
||||
min_occupancy > 0 && lower_bound_percent <= upper_bound_percent //
|
||||
&& lower_bound_percent >= cudaSharedmemCarveoutMaxL1 //
|
||||
&& upper_bound_percent <= cudaSharedmemCarveoutMaxShared);
|
||||
for (int p = lower_bound_percent; p <= upper_bound_percent; ++p) {
|
||||
set_smem_carveout(kernel_ptr, p);
|
||||
const auto occupancy = get_blocks_per_sm(kernel_ptr, block_threads, dyn_smem_bytes);
|
||||
if (occupancy >= min_occupancy) return {p, occupancy};
|
||||
}
|
||||
SGL_PANIC << "no carveout in [" << lower_bound_percent << ", " << upper_bound_percent
|
||||
<< "] can satisfy target occupancy " << min_occupancy << " for kernel " << kernel_ptr;
|
||||
__builtin_unreachable();
|
||||
#endif
|
||||
}
|
||||
|
||||
/// \brief Prefer the largest L1 carveout that keeps `kernel`'s default occupancy on the current
|
||||
/// device (PDL secondaries inherit the primary's carveout). The attribute is per device and sticky
|
||||
/// for the process; panics when no carveout restores occupancy. ROCm has no such attribute and
|
||||
/// reports percent -1. Wrap it in `init_per_device(device, ...)` to run it once per device.
|
||||
template <typename T>
|
||||
inline auto set_prefer_l1_carveout(T kernel, uint32_t block_threads, std::size_t dyn_smem_bytes = 0) -> L1Carveout {
|
||||
#ifdef USE_ROCM
|
||||
return {-1, get_blocks_per_sm(kernel, block_threads, dyn_smem_bytes)};
|
||||
#else
|
||||
const auto kernel_ptr = reinterpret_cast<const void*>(kernel);
|
||||
set_smem_carveout(kernel_ptr, cudaSharedmemCarveoutDefault);
|
||||
const auto default_occupancy = get_blocks_per_sm(kernel_ptr, block_threads, dyn_smem_bytes);
|
||||
return set_prefer_l1_carveout_with_occupancy(
|
||||
cudaSharedmemCarveoutMaxL1,
|
||||
cudaSharedmemCarveoutMaxShared,
|
||||
default_occupancy,
|
||||
kernel_ptr,
|
||||
block_threads,
|
||||
dyn_smem_bytes);
|
||||
#endif
|
||||
}
|
||||
|
||||
/// \brief Opt `kernel` into `dyn_smem_bytes` of dynamic shared memory on the current device.
|
||||
template <typename T>
|
||||
inline void set_max_dynamic_smem(T kernel, std::size_t dyn_smem_bytes) {
|
||||
const auto kernel_ptr = reinterpret_cast<const void*>(kernel);
|
||||
const auto num_bytes = static_cast<int>(dyn_smem_bytes);
|
||||
CHECK_CUDA(::cudaFuncSetAttribute(kernel_ptr, cudaFuncAttributeMaxDynamicSharedMemorySize, num_bytes));
|
||||
}
|
||||
|
||||
/// \brief Run `fn` on the first call per device, keyed off the uniqueness of (`Tag`, `Fn`).
|
||||
/// \note Relaxed on purpose, to keep the hot path a single atomic load: concurrent first callers
|
||||
/// may all run `fn`, so it must be idempotent (e.g. setting a function attribute). A throwing `fn`
|
||||
/// leaves the device uninitialized, and devices past `kNumStaticMaxDevice` rerun `fn` every call.
|
||||
template <typename Tag = void, std::invocable Fn>
|
||||
inline void init_per_device(int device_id, bool use_cache, Fn&& fn) {
|
||||
static_assert(kNumStaticMaxDevice % 8 == 0);
|
||||
constexpr uint32_t kNumBytes = kNumStaticMaxDevice / 8;
|
||||
static std::atomic_uint8_t s_initialized[kNumBytes]{};
|
||||
const auto device = static_cast<uint32_t>(device_id);
|
||||
const auto can_use_cache = use_cache && device < kNumStaticMaxDevice;
|
||||
const auto mask = static_cast<uint8_t>(1u << (device % 8));
|
||||
if (can_use_cache && s_initialized[device / 8].load() & mask) return;
|
||||
std::forward<Fn>(fn)();
|
||||
if (can_use_cache) s_initialized[device / 8].fetch_or(mask);
|
||||
}
|
||||
|
||||
// Some wrappers of the above functions
|
||||
|
||||
template <typename Tag = void, std::invocable Fn>
|
||||
inline void init_per_device(details::MaybeDevice device, Fn&& fn) {
|
||||
return init_per_device<Tag>(device.device_id, true, std::forward<Fn>(fn));
|
||||
}
|
||||
|
||||
template <auto kernel>
|
||||
inline void set_max_dynamic_smem_per_device(details::MaybeDevice device, std::size_t dyn_smem_bytes) {
|
||||
using Tag = details::Tag<kernel>;
|
||||
return init_per_device<Tag>(device.device_id, true, [=] { set_max_dynamic_smem(kernel, dyn_smem_bytes); });
|
||||
}
|
||||
|
||||
} // namespace host::runtime
|
||||
|
||||
} // namespace sglang
|
||||
|
||||
@@ -11,7 +11,6 @@
|
||||
/// - Typed `load_as` / `store_as` for void-pointer access.
|
||||
/// - `pointer::offset` for safe void-pointer arithmetic.
|
||||
/// - `host::LaunchKernel` - kernel launcher with optional PDL.
|
||||
/// - `host::prefer_l1_carveout` / `host::ensure_prefer_l1` - occupancy-preserving L1 carveout preference.
|
||||
/// - `host::RuntimeDeviceCheck` - CUDA error checking.
|
||||
|
||||
#pragma once
|
||||
@@ -25,14 +24,10 @@
|
||||
#include <concepts>
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <map>
|
||||
#include <mutex>
|
||||
#include <optional>
|
||||
#include <type_traits>
|
||||
#include <utility>
|
||||
#ifndef USE_ROCM
|
||||
#include <tvm/ffi/extra/cuda/device_guard.h>
|
||||
|
||||
#include <cuda.h>
|
||||
#include <cuda_bf16.h>
|
||||
#include <cuda_fp16.h>
|
||||
#include <cuda_fp8.h>
|
||||
@@ -59,10 +54,6 @@ inline constexpr auto cudaSuccess = hipSuccess;
|
||||
#define cudaDeviceGetAttribute hipDeviceGetAttribute
|
||||
#define cudaDevAttrComputeCapabilityMajor hipDeviceAttributeComputeCapabilityMajor
|
||||
#define cudaDevAttrComputeCapabilityMinor hipDeviceAttributeComputeCapabilityMinor
|
||||
#define cudaGetDevice hipGetDevice
|
||||
#ifndef cudaOccupancyMaxActiveBlocksPerMultiprocessor
|
||||
#define cudaOccupancyMaxActiveBlocksPerMultiprocessor hipOccupancyMaxActiveBlocksPerMultiprocessor
|
||||
#endif
|
||||
#define cudaFuncSetAttribute hipFuncSetAttribute
|
||||
#define cudaFuncAttributeMaxDynamicSharedMemorySize hipFuncAttributeMaxDynamicSharedMemorySize
|
||||
#endif
|
||||
@@ -337,13 +328,42 @@ SGL_DEVICE_HOST constexpr uint32_t get_tmem_cols(uint32_t num_cols) {
|
||||
|
||||
namespace host {
|
||||
|
||||
namespace runtime {
|
||||
|
||||
inline bool is_cuda_error(::cudaError_t error) {
|
||||
return error != ::cudaSuccess;
|
||||
}
|
||||
|
||||
inline const char* get_cuda_error_string(::cudaError_t error) {
|
||||
return ::cudaGetErrorString(error);
|
||||
}
|
||||
|
||||
#ifndef USE_ROCM
|
||||
|
||||
inline bool is_cuda_error(::CUresult error) {
|
||||
return error != ::CUDA_SUCCESS;
|
||||
}
|
||||
|
||||
inline const char* get_cuda_error_string(::CUresult error) {
|
||||
const char* error_string = "";
|
||||
// cuGetErrorString only fails with CUDA_ERROR_INVALID_VALUE, leaving the string null.
|
||||
const auto new_error = ::cuGetErrorString(error, &error_string);
|
||||
return new_error == ::CUDA_SUCCESS ? error_string : "Unknown error";
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
} // namespace runtime
|
||||
|
||||
/**
|
||||
* \brief Check the CUDA error code and panic with location info on failure.
|
||||
* \brief Check a CUDA runtime (`cudaError_t`) or driver (`CUresult`) error code
|
||||
* and panic with location info on failure.
|
||||
*/
|
||||
inline void RuntimeDeviceCheck(::cudaError_t error, DebugInfo location = {}) {
|
||||
if (error != ::cudaSuccess) {
|
||||
template <typename T>
|
||||
inline void RuntimeDeviceCheck(T error, DebugInfo location = {}) {
|
||||
if (runtime::is_cuda_error(error)) {
|
||||
[[unlikely]];
|
||||
host::panic(location, "CUDA error: ", ::cudaGetErrorString(error));
|
||||
panic(location, "CUDA error: ", runtime::get_cuda_error_string(error));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -352,58 +372,6 @@ inline void RuntimeDeviceCheck(DebugInfo location = {}) {
|
||||
return RuntimeDeviceCheck(::cudaGetLastError(), location);
|
||||
}
|
||||
|
||||
/// \brief Where `prefer_l1_carveout` settled: the carveout percentage and the kernel's blocks/SM there.
|
||||
struct L1Carveout {
|
||||
int carveout_pct;
|
||||
uint32_t blocks_per_sm;
|
||||
};
|
||||
|
||||
/// \brief Prefer the largest L1 carveout that keeps `kernel`'s default occupancy on `device_id`
|
||||
/// (PDL secondaries inherit the primary's carveout). Per device, sticky for the process; panics
|
||||
/// when no carveout restores occupancy. ROCm has no such attribute and reports pct -1.
|
||||
template <typename T>
|
||||
inline auto prefer_l1_carveout(T&& kernel, int device_id, uint32_t block_threads, std::size_t dyn_smem_bytes = 0)
|
||||
-> L1Carveout {
|
||||
const auto blocks_per_sm = [&] {
|
||||
int blocks = 0;
|
||||
RuntimeDeviceCheck(::cudaOccupancyMaxActiveBlocksPerMultiprocessor(&blocks, kernel, block_threads, dyn_smem_bytes));
|
||||
return static_cast<uint32_t>(blocks);
|
||||
};
|
||||
#ifdef USE_ROCM
|
||||
(void)device_id;
|
||||
return {-1, blocks_per_sm()};
|
||||
#else
|
||||
// The attribute and the occupancy query both act on the current device.
|
||||
tvm::ffi::CUDADeviceGuard guard(device_id);
|
||||
const auto set_carveout = [&](int pct) {
|
||||
RuntimeDeviceCheck(::cudaFuncSetAttribute(kernel, cudaFuncAttributePreferredSharedMemoryCarveout, pct));
|
||||
};
|
||||
set_carveout(cudaSharedmemCarveoutDefault);
|
||||
const uint32_t occupancy = blocks_per_sm();
|
||||
RuntimeCheck(occupancy > 0, "kernel does not fit on an SM");
|
||||
for (int pct = cudaSharedmemCarveoutMaxL1;; ++pct) {
|
||||
set_carveout(pct);
|
||||
if (const uint32_t now = blocks_per_sm(); now >= occupancy) return {pct, now};
|
||||
RuntimeCheck(pct < cudaSharedmemCarveoutMaxShared, "no carveout restores occupancy ", occupancy);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
/// \brief `prefer_l1_carveout` memoized per (kernel, device): the attribute is sticky, so a hit returns the
|
||||
/// settled value without touching the device. Load-time configure() and the launch builder share the memo.
|
||||
/// Callers must keep the block size and dynamic shared memory fixed for each (kernel, device), and
|
||||
/// must not change the kernel's carveout attribute after configuration.
|
||||
template <typename T>
|
||||
inline auto ensure_prefer_l1(T kernel, int device_id, uint32_t block_threads, std::size_t dyn_smem_bytes = 0)
|
||||
-> L1Carveout {
|
||||
static std::mutex mutex;
|
||||
static std::map<std::pair<const void*, int>, L1Carveout> settled;
|
||||
const std::lock_guard<std::mutex> lock(mutex);
|
||||
const auto key = std::make_pair(reinterpret_cast<const void*>(kernel), device_id);
|
||||
if (const auto it = settled.find(key); it != settled.end()) return it->second;
|
||||
return settled.emplace(key, prefer_l1_carveout(kernel, device_id, block_threads, dyn_smem_bytes)).first->second;
|
||||
}
|
||||
|
||||
/**
|
||||
* \brief Kernel launcher with automatic stream resolution and PDL support.
|
||||
*
|
||||
@@ -413,8 +381,6 @@ inline auto ensure_prefer_l1(T kernel, int device_id, uint32_t block_threads, st
|
||||
* .enable_pdl(true)(my_kernel, arg0, arg1);
|
||||
* host::LaunchKernel(grid, block, stream)
|
||||
* .config({.use_pdl = true, .cluster_dim = cluster_dim})(my_kernel, arg0);
|
||||
* host::LaunchKernel(grid, block, device)
|
||||
* .config({.use_pdl = true, .prefer_l1 = true})(my_kernel, arg0);
|
||||
* \endcode
|
||||
*
|
||||
* The constructor resolves the CUDA stream from a `DLDevice` (via `TVMFFIEnvGetStream`)
|
||||
@@ -425,7 +391,6 @@ struct LaunchKernel {
|
||||
struct KernelConfig {
|
||||
bool use_pdl = false;
|
||||
std::optional<dim3> cluster_dim = std::nullopt;
|
||||
bool prefer_l1 = false; // `ensure_prefer_l1` on the launch device before the first launch
|
||||
};
|
||||
|
||||
public:
|
||||
@@ -436,8 +401,7 @@ struct LaunchKernel {
|
||||
std::size_t dynamic_shared_mem_bytes = 0,
|
||||
DebugInfo location = {}) noexcept
|
||||
: m_config(s_make_config(grid_dim, block_dim, resolve_device(device), dynamic_shared_mem_bytes)),
|
||||
m_location(location),
|
||||
m_device_id(device.device_id) {}
|
||||
m_location(location) {}
|
||||
|
||||
explicit LaunchKernel(
|
||||
dim3 grid_dim,
|
||||
@@ -445,9 +409,7 @@ struct LaunchKernel {
|
||||
cudaStream_t stream,
|
||||
std::size_t dynamic_shared_mem_bytes = 0,
|
||||
DebugInfo location = {}) noexcept
|
||||
: m_config(s_make_config(grid_dim, block_dim, stream, dynamic_shared_mem_bytes)),
|
||||
m_location(location),
|
||||
m_device_id(-1) {}
|
||||
: m_config(s_make_config(grid_dim, block_dim, stream, dynamic_shared_mem_bytes)), m_location(location) {}
|
||||
|
||||
LaunchKernel(const LaunchKernel&) = delete;
|
||||
LaunchKernel& operator=(const LaunchKernel&) = delete;
|
||||
@@ -483,11 +445,6 @@ struct LaunchKernel {
|
||||
return *this;
|
||||
}
|
||||
|
||||
auto prefer_l1(bool enabled = true) -> LaunchKernel& {
|
||||
m_prefer_l1 = enabled;
|
||||
return *this;
|
||||
}
|
||||
|
||||
/**
|
||||
* \brief Configure the kernel launch with the given options.
|
||||
* \param config The kernel configuration options.
|
||||
@@ -499,13 +456,11 @@ struct LaunchKernel {
|
||||
auto config(const KernelConfig& config) -> LaunchKernel& {
|
||||
if (config.use_pdl) this->enable_pdl(true);
|
||||
if (config.cluster_dim) this->enable_cluster(*config.cluster_dim);
|
||||
if (config.prefer_l1) this->prefer_l1(true);
|
||||
return *this;
|
||||
}
|
||||
|
||||
template <typename T, typename... Args>
|
||||
auto operator()(T&& kernel, Args&&... args) const -> void {
|
||||
if (m_prefer_l1) apply_prefer_l1(kernel);
|
||||
#ifdef USE_ROCM
|
||||
hipLaunchKernelGGL(
|
||||
std::forward<T>(kernel),
|
||||
@@ -540,28 +495,15 @@ struct LaunchKernel {
|
||||
return config;
|
||||
}
|
||||
|
||||
// Memo hit after load-time configure(); stream-constructed launches use the current device.
|
||||
template <typename T>
|
||||
void apply_prefer_l1(T&& kernel) const {
|
||||
int device_id = m_device_id;
|
||||
if (device_id < 0) RuntimeDeviceCheck(::cudaGetDevice(&device_id));
|
||||
const dim3 block = m_config.blockDim;
|
||||
ensure_prefer_l1(+kernel, device_id, block.x * block.y * block.z, m_config.dynamicSmemBytes);
|
||||
}
|
||||
|
||||
cudaLaunchConfig_t m_config;
|
||||
const DebugInfo m_location;
|
||||
cudaLaunchAttribute m_attrs[2];
|
||||
int m_device_id;
|
||||
bool m_prefer_l1 = false;
|
||||
};
|
||||
|
||||
// The empty-true-branch if/else form keeps a trailing `else` in user code
|
||||
// bound to the user's `if`, not to the macro's.
|
||||
#define CHECK_CUDA(COND) \
|
||||
if (const auto error = (COND); error == ::cudaSuccess) [[likely]] { \
|
||||
} else \
|
||||
host::Error() << "CUDA error: " << ::cudaGetErrorString(error) << ". "
|
||||
#define CHECK_CUDA(COND) \
|
||||
if (const auto _err = (COND); !::sglang::host::runtime::is_cuda_error(_err)) [[likely]] { \
|
||||
} else \
|
||||
SGL_PANIC << "CUDA error: " << ::sglang::host::runtime::get_cuda_error_string(_err) << ". "
|
||||
|
||||
} // namespace host
|
||||
|
||||
|
||||
@@ -178,37 +178,28 @@ inline auto irange(T start, T end) {
|
||||
|
||||
/** \brief Error class for stream-style error logging. */
|
||||
struct Error {
|
||||
Error(DebugInfo location = {}) {
|
||||
m_oss << "Failed at " << location.file_name() << ":" << location.line() << ": ";
|
||||
std::ostringstream stream;
|
||||
explicit Error(DebugInfo location = {}) {
|
||||
stream << "Failed at " << location.file_name() << ":" << location.line() << ": ";
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
Error& operator<<(T&& arg) {
|
||||
m_oss << std::forward<T>(arg);
|
||||
return *this;
|
||||
}
|
||||
|
||||
[[noreturn]]
|
||||
~Error() noexcept(false) {
|
||||
throw PanicError(std::move(m_oss).str());
|
||||
throw PanicError(std::move(stream).str());
|
||||
}
|
||||
|
||||
private:
|
||||
std::ostringstream m_oss;
|
||||
};
|
||||
|
||||
#define SGL_PANIC ::sglang::host::Error().stream
|
||||
|
||||
/**
|
||||
* \brief 0-overhead CHECK macro for host code. This can avoid unnecessary
|
||||
* instantiation of error messages when the condition is true.
|
||||
*
|
||||
* Usage: CHECK_HOST(ptr != nullptr) << "Pointer must not be null";
|
||||
*/
|
||||
// The empty-true-branch if/else form keeps a trailing `else` in user code
|
||||
// bound to the user's `if`, not to the macro's.
|
||||
#define CHECK_HOST(COND) \
|
||||
if (COND) [[likely]] { \
|
||||
} else \
|
||||
host::Error()
|
||||
SGL_PANIC
|
||||
|
||||
} // namespace host
|
||||
|
||||
|
||||
@@ -353,6 +353,8 @@ class Envs:
|
||||
SGLANG_LOG_REQUEST_HEADERS = EnvTuple(tuple())
|
||||
SGLANG_LOG_SCHEDULER_STATUS_TARGET = EnvStr("")
|
||||
SGLANG_LOG_SCHEDULER_STATUS_INTERVAL = EnvFloat(60.0)
|
||||
# Read by JIT kernel headers (sgl_kernel/logging.h): DEBUG | INFO | WARNING | ERROR.
|
||||
SGLANG_JIT_LOG_LEVEL = EnvStr("WARNING")
|
||||
SGLANG_ENABLE_RANK_CONSENSUS_CHECKER = EnvBool(False)
|
||||
|
||||
# ===================================================================
|
||||
|
||||
@@ -1,64 +0,0 @@
|
||||
#include <sgl_kernel/tensor.h>
|
||||
#include <sgl_kernel/utils.cuh>
|
||||
|
||||
namespace sglang {
|
||||
|
||||
template <int kSmem, int kMode>
|
||||
__global__ void carveout_probe(int32_t *output) {
|
||||
device::PDLWaitPrimary<kMode == 2>();
|
||||
extern __shared__ int32_t scratch[];
|
||||
if constexpr (kSmem > 0) {
|
||||
scratch[threadIdx.x] = threadIdx.x;
|
||||
__syncthreads();
|
||||
output[threadIdx.x] = scratch[255 - threadIdx.x];
|
||||
} else {
|
||||
output[threadIdx.x] = 255 - threadIdx.x;
|
||||
}
|
||||
}
|
||||
|
||||
template <int kSmem, int kMode>
|
||||
void check_launch_carveout(tvm::ffi::TensorView output) {
|
||||
host::TensorMatcher({256})
|
||||
.with_dtype<int32_t>()
|
||||
.with_device<kDLCUDA>()
|
||||
.verify(output);
|
||||
const auto kernel = carveout_probe<kSmem, kMode>;
|
||||
const int device_id = output.device().device_id;
|
||||
const tvm::ffi::CUDADeviceGuard guard(device_id);
|
||||
int baseline = 0;
|
||||
CHECK_CUDA(cudaOccupancyMaxActiveBlocksPerMultiprocessor(&baseline, kernel,
|
||||
256, kSmem));
|
||||
CHECK_HOST(baseline > 0);
|
||||
if constexpr (kMode == 2)
|
||||
host::ensure_prefer_l1(kernel, device_id, 256, kSmem);
|
||||
auto *ptr = static_cast<int32_t *>(output.data_ptr());
|
||||
if constexpr (kMode == 0) {
|
||||
host::LaunchKernel(1, 256, output.device(), kSmem)(kernel, ptr);
|
||||
} else if constexpr (kMode == 1) {
|
||||
host::LaunchKernel(
|
||||
1, 256, host::LaunchKernel::resolve_device(output.device()), kSmem)
|
||||
.prefer_l1()(kernel, ptr);
|
||||
} else {
|
||||
host::LaunchKernel(1, 256, output.device(), kSmem)
|
||||
.config({.use_pdl = true, .prefer_l1 = true})(kernel, ptr);
|
||||
}
|
||||
cudaFuncAttributes attrs{};
|
||||
CHECK_CUDA(cudaFuncGetAttributes(&attrs, kernel));
|
||||
if constexpr (kMode == 0) {
|
||||
CHECK_HOST(attrs.preferredShmemCarveout == cudaSharedmemCarveoutDefault);
|
||||
} else {
|
||||
const auto receipt = host::ensure_prefer_l1(kernel, device_id, 256, kSmem);
|
||||
const auto memo = host::ensure_prefer_l1(kernel, device_id, 256, kSmem);
|
||||
CHECK_HOST(receipt.carveout_pct >= 0 && receipt.carveout_pct <= 100);
|
||||
CHECK_HOST(receipt.blocks_per_sm >= static_cast<uint32_t>(baseline));
|
||||
CHECK_HOST(attrs.preferredShmemCarveout == receipt.carveout_pct);
|
||||
CHECK_HOST(memo.carveout_pct == receipt.carveout_pct &&
|
||||
memo.blocks_per_sm == receipt.blocks_per_sm);
|
||||
int actual = 0;
|
||||
CHECK_CUDA(cudaOccupancyMaxActiveBlocksPerMultiprocessor(&actual, kernel,
|
||||
256, kSmem));
|
||||
CHECK_HOST(actual >= baseline);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace sglang
|
||||
@@ -1,45 +0,0 @@
|
||||
"""L1 preference must retain default occupancy and leave other devices untouched."""
|
||||
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from sglang.kernels.jit.utils import load_jit
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=10, stage="base-b-kernel-unit", runner_config="1-gpu-large")
|
||||
|
||||
|
||||
def test_launch_carveout():
|
||||
"""A MaxL1-only policy loses occupancy for shared-memory-heavy kernels."""
|
||||
current = torch.cuda.current_device()
|
||||
untouched = [
|
||||
device
|
||||
for device in range(torch.cuda.device_count())
|
||||
if device != current and not torch._C._cuda_hasPrimaryContext(device)
|
||||
]
|
||||
cases = [(smem, mode) for smem in (0, 32768) for mode in range(3)]
|
||||
module = load_jit(
|
||||
"test_launch_carveout",
|
||||
cuda_files=[str(pathlib.Path(__file__).with_name("launch_carveout.cuh"))],
|
||||
cuda_wrappers=[
|
||||
(f"run_{smem}_{mode}", f"check_launch_carveout<{smem}, {mode}>")
|
||||
for smem, mode in cases
|
||||
],
|
||||
)
|
||||
expected = torch.arange(255, -1, -1, dtype=torch.int32, device="cuda")
|
||||
output = torch.empty_like(expected)
|
||||
for smem, mode in cases:
|
||||
run = module[f"run_{smem}_{mode}"]
|
||||
for _ in range(2):
|
||||
output.fill_(-1)
|
||||
run(output)
|
||||
assert torch.equal(output, expected)
|
||||
for device in untouched:
|
||||
assert not torch._C._cuda_hasPrimaryContext(device)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(pytest.main([__file__]))
|
||||
Reference in New Issue
Block a user