[JIT] Clean up kernel headers: move runtime helpers, add once.h (#41147)

Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ziyi Xu
2026-10-14 10:46:51 +08:00
committed by GitHub
co-authored by Claude Opus 5.5
parent b50fdeb0ef
commit 4f5fcbfef5
20 changed files with 436 additions and 348 deletions
@@ -29,12 +29,12 @@ using F2S_T = int64_t;
using IDX_T = int64_t;
/// NOTE: for the internal use, we pack the ragged and batch id, since both not exceed 65536
SGL_DEVICE __host__ PlanW pack_w(uint32_t ragged_id, uint32_t batch_id, int32_t seq_len) {
SGL_DEVICE_HOST PlanW pack_w(uint32_t ragged_id, uint32_t batch_id, int32_t seq_len) {
return {static_cast<uint32_t>(ragged_id | batch_id << 16), seq_len};
}
/// NOTE: for the internal use, we pack the ragged and batch id, since both not exceed 65536
SGL_DEVICE uint2 unpack_w(PlanW plan) {
SGL_DEVICE_HOST uint2 unpack_w(PlanW plan) {
return {static_cast<uint16_t>(plan.ragged_id), static_cast<uint16_t>(plan.ragged_id >> 16)};
}
@@ -85,12 +85,6 @@ struct TopKBF16Params {
uint32_t page_bits;
};
SGL_DEVICE uint32_t get_ptx_lane_id() {
uint32_t lane_id;
asm volatile("mov.u32 %0, %%laneid;" : "=r"(lane_id));
return lane_id;
}
/// \brief Exclusive suffix scan: lane `L` gets the sum over lanes `> L`.
SGL_DEVICE uint32_t warp_exclusive_suffix_sum(uint32_t x, uint32_t lane_id) {
uint32_t inc = x;
@@ -183,7 +177,7 @@ __global__ __launch_bounds__(TopKBF16Config::kBlockSize, TopKBF16Config::kOccupa
const auto bx = blockIdx.x;
const auto tx = threadIdx.x;
const auto lane_id = get_ptx_lane_id();
const auto lane_id = warp::get_lane_id();
const auto warp_id = tx / kWarpThreads;
const auto topk = params.topk;
// a selected index i maps through this row's table to slot
@@ -1,6 +1,7 @@
#include <sgl_kernel/tensor.h>
#include <sgl_kernel/utils.h>
#include <sgl_kernel/runtime.cuh>
#include <sgl_kernel/utils.cuh>
#include <dlpack/dlpack.h>
@@ -14,8 +15,8 @@ namespace sglang {
// `topk` is a *runtime* value (<= kMaxTopK), so one module serves every k. It
// used to be baked in via -DSGL_TOPK, which built a separate module per k --
// and because `kTopK` came from a macro rather than a template parameter, both
// modules exported identically mangled symbols. The function-local static in
// setup_kernel_smem_once() is emitted as STB_GNU_UNIQUE, which the loader
// modules exported identically mangled symbols. The function-local static that
// guarded the smem opt-in is emitted as STB_GNU_UNIQUE, which the loader
// merges across every loaded object, so whichever module was used second
// skipped its cudaFuncSetAttribute opt-in and then failed to launch with 64 KB
// of dynamic shared memory ("invalid argument").
@@ -270,16 +271,6 @@ __global__ void topk_transform_kernel(const __grid_constant__ TopKParams params)
device::PDLTriggerSecondary<kUsePDL>();
}
template <auto* f, size_t kMaxDynamicSMEM>
void setup_kernel_smem_once(host::DebugInfo where = {}) {
[[maybe_unused]]
static const auto result = [] {
const auto fptr = std::bit_cast<const void*>(f);
return ::cudaFuncSetAttribute(fptr, ::cudaFuncAttributeMaxDynamicSharedMemorySize, kMaxDynamicSMEM);
}();
host::RuntimeDeviceCheck(result, where);
}
template <bool kUsePDL>
struct TopKKernel {
static constexpr auto kernel = topk_transform_kernel<kUsePDL>;
@@ -344,7 +335,7 @@ struct TopKKernel {
.topk = topk,
};
constexpr auto kSMEM_ = kSMEM + sizeof(int32_t); // align up a little
setup_kernel_smem_once<kernel, kSMEM_>();
runtime::set_max_dynamic_smem_per_device<kernel>(device.unwrap(), kSMEM_);
LaunchKernel(batch_size, kTopKBlockSize, device.unwrap(), kSMEM_).enable_pdl(kUsePDL)(kernel, params);
}
};
@@ -12,6 +12,7 @@
#include <sgl_kernel/tensor.h>
#include <sgl_kernel/utils.h>
#include <sgl_kernel/runtime.cuh>
#include <sgl_kernel/utils.cuh>
#include <sgl_kernel/deepseek_v4/topk_impl.cuh>
@@ -1103,12 +1104,10 @@ struct TopKKernel {
constexpr uint32_t kClusterSize = 16;
// Widths above 8 are non-portable; the launch is rejected without this.
const auto kernel = topk_small_batch_cluster_kernel<kUsePDL, kMode, kClusterSize, 1>;
[[maybe_unused]]
static const bool _ = [&kernel] {
const auto kernel_ptr = reinterpret_cast<const void*>(kernel);
const auto kernel_ptr = reinterpret_cast<const void*>(kernel);
runtime::init_per_device(device, [kernel_ptr] {
CHECK_CUDA(::cudaFuncSetAttribute(kernel_ptr, ::cudaFuncAttributeNonPortableClusterSizeAllowed, 1));
return true;
}();
});
return LaunchKernel({batch_size, kClusterSize}, kBlockSize, device)
.config({.use_pdl = kUsePDL, .cluster_dim = dim3{1, kClusterSize}})
.launch(kernel, params);
@@ -22,6 +22,7 @@
#include <sgl_kernel/utils.h>
#include <sgl_kernel/mbarrier.cuh>
#include <sgl_kernel/runtime.cuh>
#include <sgl_kernel/type.cuh>
#include <sgl_kernel/utils.cuh>
#include <sgl_kernel/warp.cuh>
@@ -779,11 +780,10 @@ inline void wo_a_fused_run(
const auto launch = [&](const auto* pos) {
using PosT = std::decay_t<decltype(*pos)>;
constexpr auto kernel = wo_a_fused_kernel<Trait, PosT>;
[[maybe_unused]] static const auto _ = [] {
CHECK_CUDA(cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, sizeof(Smem)));
CHECK_CUDA(cudaFuncSetAttribute(kernel, cudaFuncAttributePreferredSharedMemoryCarveout, 100));
return 0;
}();
runtime::init_per_device(device, [=] {
runtime::set_max_dynamic_smem(kernel, sizeof(Smem));
runtime::set_smem_carveout(kernel, cudaSharedmemCarveoutMaxShared);
});
LaunchKernel(kClusters * kSplitK, kThreads, device, sizeof(Smem))
.config({.use_pdl = true, .cluster_dim = dim3{kSplitK, 1, 1}})(
kernel,
@@ -3,12 +3,12 @@
#include <sgl_kernel/tensor.h>
#include <sgl_kernel/utils.h>
#include <sgl_kernel/runtime.cuh>
#include <sgl_kernel/utils.cuh>
#include <dlpack/dlpack.h>
#include <tvm/ffi/container/tensor.h>
#include <bit>
#include <cstddef>
#include <cstdint>
#ifndef USE_ROCM
@@ -354,20 +354,6 @@ __global__ __launch_bounds__(kThreadsPerBlock) void kpool_topk_transform_kernel(
}
}
template <auto* f, std::size_t kMaxDynamicSMEM>
void setup_kernel_smem_once(host::DebugInfo where = {}) {
[[maybe_unused]]
static const auto result = [] {
const auto fptr = std::bit_cast<const void*>(f);
#if defined(__HIP_PLATFORM_AMD__)
return ::hipFuncSetAttribute(fptr, ::hipFuncAttributeMaxDynamicSharedMemorySize, kMaxDynamicSMEM);
#else
return ::cudaFuncSetAttribute(fptr, ::cudaFuncAttributeMaxDynamicSharedMemorySize, kMaxDynamicSMEM);
#endif
}();
host::RuntimeDeviceCheck(result, where);
}
template <typename T>
const T* optional_data_ptr(const tvm::ffi::Optional<tvm::ffi::TensorView>& opt) {
if (!opt.has_value()) {
@@ -463,7 +449,7 @@ struct KpoolTopKTransformKernel {
.input_stride = static_cast<int64_t>(S.unwrap()),
};
setup_kernel_smem_once<kernel, kSmem>();
runtime::set_max_dynamic_smem_per_device<kernel>(device.unwrap().device_id, kSmem);
LaunchKernel(batch_size, kThreadsPerBlock, device.unwrap(), kSmem)(
kernel,
params,
@@ -48,6 +48,7 @@ limitations under the License.
#include <cutlass/util/packed_stride.hpp>
#include <dlpack/dlpack.h>
#include <tvm/ffi/container/tensor.h>
#include <tvm/ffi/extra/cuda/device_guard.h>
#include "cutlass_extensions/epilogue/epilogue_per_row_per_col_scale.h"
#include "cutlass_extensions/gemm/gemm_universal_base_compat.h"
@@ -21,6 +21,7 @@
#include <sgl_kernel/tensor.h>
#include <sgl_kernel/utils.h>
#include <sgl_kernel/runtime.cuh>
#include <sgl_kernel/utils.cuh>
#include <tvm/ffi/container/tensor.h>
@@ -593,9 +594,9 @@ void invokeFusedAGemm(T* output, T const* mat_a, T const* mat_b, int num_tokens,
dim3 grid(cta_m_cnt, cta_n_cnt, 1);
dim3 block_size(256);
auto kernel = fused_a_gemm_kernel<batch_size, gemm_m, gemm_k, tile_m, tile_n, tile_k, stage_cnt>;
if (smem_bytes >= (48 * 1024)) {
host::RuntimeDeviceCheck(cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, smem_bytes));
constexpr auto kernel = fused_a_gemm_kernel<batch_size, gemm_m, gemm_k, tile_m, tile_n, tile_k, stage_cnt>;
if constexpr (smem_bytes >= (48 * 1024)) {
host::runtime::set_max_dynamic_smem_per_device<kernel>(device.device_id, smem_bytes);
}
host::LaunchKernel(grid, block_size, device, smem_bytes).enable_pdl(kUsePDL)(kernel, output, mat_a, mat_b, gemm_n);
}
@@ -28,6 +28,7 @@
#include <sgl_kernel/utils.h>
#include <sgl_kernel/mbarrier.cuh>
#include <sgl_kernel/runtime.cuh>
#include <sgl_kernel/utils.cuh>
#include <sgl_kernel/warp.cuh>
@@ -130,7 +131,7 @@ inline constexpr uint32_t kHicacheTmaMaxMapRowBytes = 256 * 8;
// Rows per chunk: largest power of two that fits the stage, so chunks never
// straddle a (power-of-two) page and a page run stays one bulk op.
__host__ __device__ constexpr uint32_t hicache_tma_rows_per_chunk(uint32_t stage_bytes, uint32_t row_bytes) {
SGL_DEVICE_HOST constexpr uint32_t hicache_tma_rows_per_chunk(uint32_t stage_bytes, uint32_t row_bytes) {
uint32_t rows = 1;
while (rows * 2 <= stage_bytes / row_bytes && rows * 2 <= kHicacheTmaMaxRows)
rows *= 2;
@@ -438,16 +439,12 @@ struct HiCacheTmaKernel {
(params.has_v ? 2u : 1u) * params.num_layers * div_ceil(params.length, rows_per_chunk(params.row_bytes));
constexpr std::size_t kSmemBytes = sizeof(Smem);
static const bool attr_set = [] {
for (auto fn : {kernel<int32_t>, kernel<int64_t>}) {
RuntimeDeviceCheck(
cudaFuncSetAttribute(fn, cudaFuncAttributeMaxDynamicSharedMemorySize, static_cast<int>(kSmemBytes)));
}
return true;
}();
(void)attr_set;
LaunchKernel(std::min(chunks, kBlockQuota), kThreads, device, kSmemBytes)(
use_int32 ? kernel<int32_t> : kernel<int64_t>, params);
runtime::init_per_device(device, [=] {
runtime::set_max_dynamic_smem(kernel<int32_t>, kSmemBytes);
runtime::set_max_dynamic_smem(kernel<int64_t>, kSmemBytes);
});
const auto fn = use_int32 ? kernel<int32_t> : kernel<int64_t>;
LaunchKernel(std::min(chunks, kBlockQuota), kThreads, device, kSmemBytes)(fn, params);
}
// Cache operand viewed as [-1, D] rows; binds row dim, stride and dtype.
@@ -779,7 +779,10 @@ void load_cache_to_device_buffer(
constexpr size_t smem_bytes = SmemLayout<NUM_TOP_K_TOKENS, HOT_BUFFER_SIZE>::BYTES;
#ifndef USE_ROCM
if constexpr (smem_bytes > 48u * 1024u) {
cudaFuncSetAttribute(kernel_fn, cudaFuncAttributeMaxDynamicSharedMemorySize, smem_bytes);
/// NOTE: for each launch template, the `kernel_fn` must be unique
runtime::init_per_device(kernel_device, [=] { //
runtime::set_max_dynamic_smem(kernel_fn, smem_bytes);
});
}
#endif
LaunchKernel(bs, BLOCK_SIZE, device, smem_bytes)(
@@ -18,6 +18,7 @@
#include <sgl_kernel/tensor.h>
#include <sgl_kernel/runtime.cuh>
#include <sgl_kernel/utils.cuh>
#include <sgl_kernel/speculative/sampling.cuh>
@@ -78,7 +79,7 @@ void tree_speculative_sampling_target_only(
constexpr uint32_t block_threads = 1024;
constexpr size_t smem_size = sizeof(SamplingTempStorage<block_threads, SCAN_ALGO, REDUCE_ALGO>);
auto kernel = speculative_sampling::TreeSpeculativeSamplingTargetOnly<
constexpr auto kernel = speculative_sampling::TreeSpeculativeSamplingTargetOnly<
block_threads,
SCAN_ALGO,
REDUCE_ALGO,
@@ -87,7 +88,7 @@ void tree_speculative_sampling_target_only(
float,
int32_t,
int64_t>;
CHECK_CUDA(cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, smem_size));
runtime::set_max_dynamic_smem_per_device<kernel>(device.unwrap().device_id, smem_size);
LaunchKernel(static_cast<uint32_t>(batch_size.unwrap()), block_threads, device.unwrap(), smem_size)(
kernel,
static_cast<int32_t*>(predicts.data_ptr()),
@@ -1,40 +0,0 @@
#pragma once
#include <concepts>
#include <mutex>
#include <type_traits>
#include <unordered_map>
namespace sglang::host {
/**
* \brief Allocate only once for a given function.
* \tparam kThreadSafe Whether to make the allocation thread-safe.
* \tparam Salt A salt type to avoid cache collision.
* \param key The key to identify the allocation. It should be unique for each allocation.
* \param callback The callback function to perform the allocation. It should return the allocated value.
* \note The `Fn` type must be unique. It's typically a lambda type that's evaluated only once.
* Otherwise, different call-sites may hit the same cache entry.
* In case where `Fn` is not unique (e.g. std::function), make `Salt` unique to avoid cache collision.
*/
template <bool kThreadSafe = true, typename Salt = void, typename Key, std::invocable Fn>
inline auto allocate_once(Key&& key, Fn&& callback) -> std::decay_t<std::invoke_result_t<Fn>>& {
using Value = std::decay_t<std::invoke_result_t<Fn>>;
static std::unordered_map<std::decay_t<Key>, Value> s_map;
const auto alloc = [&]() -> Value& {
const auto iter = s_map.find(key);
if (iter != s_map.end()) return iter->second;
// Evaluate the callback before inserting, so a throwing callback leaves no empty entry behind.
auto value = std::forward<Fn>(callback)();
return s_map.emplace(std::forward<Key>(key), std::move(value)).first->second;
};
if constexpr (kThreadSafe) {
static std::mutex s_mutex;
const auto lock = std::lock_guard{s_mutex};
return alloc();
} else {
return alloc();
}
}
} // namespace sglang::host
@@ -0,0 +1,54 @@
#pragma once
#include <sgl_kernel/utils.h>
#include <cstdint>
#include <cstdlib>
#include <iostream>
#include <sstream>
#include <string_view>
namespace sglang::host {
enum class LogLevel : uint8_t {
DEBUG_ = 0,
INFO_ = 1,
WARNING_ = 2,
ERROR_ = 3,
DEFAULT = WARNING_,
};
inline const LogLevel kLogLevel = [] {
const auto env = std::getenv("SGLANG_JIT_LOG_LEVEL");
if (!env) return LogLevel::DEFAULT;
const auto str = std::string_view{env};
if (str == "DEBUG" || str == "debug") return LogLevel::DEBUG_;
if (str == "INFO" || str == "info") return LogLevel::INFO_;
if (str == "WARNING" || str == "warning") return LogLevel::WARNING_;
if (str == "ERROR" || str == "error") return LogLevel::ERROR_;
std::clog << "[WARNING] unrecognized SGLANG_JIT_LOG_LEVEL value '" << str << "'; using default log level."
<< std::endl;
return LogLevel::DEFAULT;
}();
struct Logger {
std::ostringstream stream;
explicit Logger(LogLevel level, DebugInfo location = {}) {
static const std::string_view kPrefix[] = {"[DEBUG]", "[INFO]", "[WARNING]", "[ERROR]"};
stream << kPrefix[static_cast<size_t>(level)] << " [" << location.file_name() << ":" << location.line() << "] ";
}
~Logger() {
std::clog << std::move(stream).str() << std::endl;
}
};
#define SGL_LOG(LEVEL) \
if (const auto _log_level = ::sglang::host::LogLevel::LEVEL; _log_level < ::sglang::host::kLogLevel) { \
} else \
::sglang::host::Logger(_log_level).stream
#define SGL_LOG_DEBUG SGL_LOG(DEBUG_)
#define SGL_LOG_INFO SGL_LOG(INFO_)
#define SGL_LOG_WARNING SGL_LOG(WARNING_)
} // namespace sglang::host
@@ -0,0 +1,147 @@
/// \file once.h
/// \brief Host-side "run it only once" helpers.
///
/// C++ spells this `std::call_once` plus a caller-provided `std::once_flag`. The helpers here
/// hide the flag inside the function, keyed off the uniqueness of the callable's type, and hand
/// back the value the callable produced. They come in two forms: one value per call-site, or
/// one value per (call-site, key).
#pragma once
#include <concepts>
#include <cstddef>
#include <functional>
#include <mutex>
#include <tuple>
#include <type_traits>
#include <unordered_map>
#include <unordered_set>
#include <utility>
namespace sglang::host {
namespace details {
/// \brief Fold any number of sub-hashes into one; the mixing step is boost's.
/// Defined for zero parts too, so an empty tuple key hashes without a special case.
inline std::size_t hash_combine(std::same_as<std::size_t> auto... parts) {
std::size_t seed = 0;
((seed ^= parts + 0x9e3779b97f4a7c15ULL + (seed << 6) + (seed >> 2)), ...);
return seed;
}
/// \brief Hash for keys `std::hash` does not cover: pairs and tuples, recursively.
struct GenericHash {
template <typename T>
std::size_t operator()(const T& value) const {
return std::hash<T>{}(value);
}
template <typename T, typename U>
std::size_t operator()(const std::pair<T, U>& value) const {
return hash_combine((*this)(value.first), (*this)(value.second));
}
template <typename... Args>
std::size_t operator()(const std::tuple<Args...>& value) const {
return std::apply([this](const auto&... args) { return hash_combine((*this)(args)...); }, value);
}
};
struct DummyMutex {
void lock() {}
void unlock() {}
};
} // namespace details
template <typename Key, typename Value, bool kThreadSafe = true>
struct CacheMap {
public:
CacheMap() = default;
CacheMap(const CacheMap&) = delete;
CacheMap& operator=(const CacheMap&) = delete;
template <typename Fn, typename KeyArg>
auto get_cached(KeyArg&& key, Fn&& factory) -> Value& {
const auto lock = std::lock_guard{m_mutex};
if (const auto iter = m_map.find(key); iter != m_map.end()) return iter->second;
// Evaluate the factory before inserting, so a throwing factory leaves no entry behind.
auto value = std::forward<Fn>(factory)();
return m_map.try_emplace(std::forward<KeyArg>(key), std::move(value)).first->second;
}
private:
[[no_unique_address]]
std::conditional_t<kThreadSafe, std::mutex, details::DummyMutex> m_mutex;
std::unordered_map<Key, Value, details::GenericHash> m_map;
};
template <typename Key, bool kThreadSafe>
struct CacheMap<Key, void, kThreadSafe> {
public:
CacheMap() = default;
CacheMap(const CacheMap&) = delete;
CacheMap& operator=(const CacheMap&) = delete;
template <typename Fn, typename KeyArg>
void get_cached(KeyArg&& key, Fn&& factory) {
const auto lock = std::lock_guard{m_mutex};
if (m_set.contains(key)) return;
// Run the factory before inserting, so a throwing factory is retried on the next call.
std::forward<Fn>(factory)();
m_set.emplace(std::forward<KeyArg>(key));
}
private:
[[no_unique_address]]
std::conditional_t<kThreadSafe, std::mutex, details::DummyMutex> m_mutex;
std::unordered_set<Key, details::GenericHash> m_set;
};
template <typename Key, bool kThreadSafe = true>
using CacheSet = CacheMap<Key, void, kThreadSafe>;
/**
* \brief A keyed `std::call_once` that also hands back the value.
* Runs `factory` at most once per (call-site, key) and returns a stable reference to its result.
* \tparam Tag A tag type to avoid cache collision.
* \tparam kThreadSafe Whether to make the initialization thread-safe. Turn it off only where the
* call-site is provably single-threaded: the lock is taken on every call, cache hits included.
* \param key The key to identify the value. Each distinct key is initialized independently.
* \param factory The function to produce the value on first use. It should return the value.
* \note The `Fn` type must be unique. It's typically a lambda type that's evaluated only once.
* Otherwise, different call-sites may hit the same cache entry.
* In case where `Fn` is not unique (e.g. std::function), make `Tag` unique to avoid cache collision.
* \note `factory` runs with the lock held, so it must not re-enter this same instantiation.
*/
template <typename Tag = void, bool kThreadSafe = true, typename Key, std::invocable Fn>
inline auto init_once(Key&& key, Fn&& factory) -> decltype(auto) {
static CacheMap<std::decay_t<Key>, std::decay_t<std::invoke_result_t<Fn>>, kThreadSafe> s_cache;
return s_cache.get_cached(std::forward<Key>(key), std::forward<Fn>(factory));
}
/**
* \brief `std::call_once` without the caller-provided `once_flag`, also handing back the value.
* Runs `factory` at most once per call-site and returns a stable reference to its result.
* \tparam Tag A tag type to avoid cache collision.
* \param factory The function to produce the value on first use. It should return the value.
* \note The `Fn` type must be unique, for the same reason as the keyed overload above: the static
* lives in the instantiation, so a non-unique `Fn` (e.g. std::function) makes two call-sites share
* one value. Make `Tag` unique in that case.
* \note Always thread-safe, and deliberately has no `kThreadSafe` knob: C++ already guarantees that
* concurrent callers block until the first one finishes, and the post-initialization fast path is
* just a guard check, so there is no lock worth skipping.
* \note If `factory` throws, the value stays uninitialized and the next call retries, matching the
* keyed overload. `factory` must not re-enter this same instantiation; that is undefined behavior.
*/
template <typename Tag = void, std::invocable Fn>
inline auto init_once(Fn&& factory) -> decltype(auto) {
if constexpr (std::is_same_v<std::invoke_result_t<Fn>, void>) {
static std::once_flag s_once;
return std::call_once(s_once, std::forward<Fn>(factory));
} else {
static auto result = std::forward<Fn>(factory)();
return (result); // parenthesized: `decltype(auto)` then deduces a reference, not a copy
}
}
} // namespace sglang::host
@@ -6,13 +6,19 @@
#pragma once
#include <sgl_kernel/utils.h>
#include <sgl_kernel/utils.cuh>
#include <dlpack/dlpack.h>
#include <tvm/ffi/container/tensor.h>
#include <atomic>
#include <concepts>
#include <cstddef>
#include <cstdint>
#include <utility>
#ifndef USE_ROCM
#include <cuda_runtime.h>
#else
@@ -59,21 +65,22 @@ namespace sglang {
namespace host::runtime {
inline void* get_device_accessible_ptr(const tvm::ffi::TensorView& tensor) {
void* ptr = tensor.data_ptr();
const auto tensor_device_type = tensor.device().device_type;
if (tensor_device_type != kDLCPU && tensor_device_type != kDLGPUHost) {
return ptr;
}
namespace details {
void* device_ptr = nullptr;
#ifdef USE_ROCM
RuntimeDeviceCheck(::hipHostGetDevicePointer(&device_ptr, ptr, 0));
#else
RuntimeDeviceCheck(::cudaHostGetDevicePointer(&device_ptr, ptr, 0));
#endif
return device_ptr;
}
template <auto V>
struct Tag {};
struct MaybeDevice {
MaybeDevice(DLDevice device) : device_id(device.device_id) {}
MaybeDevice(int32_t device_id_) : device_id(device_id_) {}
uint32_t device_id;
};
} // namespace details
// Generous bound on the device ordinals one process can see; a larger ordinal
// is not an error, it just falls through to the driver query uncached.
inline constexpr uint32_t kNumStaticMaxDevice = 72;
/**
* \brief Memo for a per-device driver query, keyed by device ordinal.
@@ -85,10 +92,6 @@ inline void* get_device_accessible_ptr(const tvm::ffi::TensorView& tensor) {
template <typename T, T kDefault>
struct DeviceCacheMap {
public:
// Generous bound on the device ordinals one process can see; a larger ordinal
// is not an error, it just falls through to the driver query uncached.
static constexpr uint32_t kNumStaticMaxDevice = 72;
constexpr DeviceCacheMap() {
for (uint32_t i = 0; i < kNumStaticMaxDevice; ++i) {
m_data[i] = kDefault;
@@ -108,13 +111,34 @@ struct DeviceCacheMap {
return value;
}
template <typename Fn>
T get_cached(int32_t device_, Fn&& fn) {
return this->get_cached(device_, true, std::forward<Fn>(fn));
}
private:
T m_data[kNumStaticMaxDevice];
};
inline void* get_device_accessible_ptr(const tvm::ffi::TensorView& tensor) {
void* ptr = tensor.data_ptr();
const auto tensor_device_type = tensor.device().device_type;
if (tensor_device_type != kDLCPU && tensor_device_type != kDLGPUHost) {
return ptr;
}
void* device_ptr = nullptr;
#ifdef USE_ROCM
CHECK_CUDA(::hipHostGetDevicePointer(&device_ptr, ptr, 0));
#else
CHECK_CUDA(::cudaHostGetDevicePointer(&device_ptr, ptr, 0));
#endif
return device_ptr;
}
// Return the maximum number of active blocks per SM for the given kernel
template <typename T>
inline auto get_blocks_per_sm(T&& kernel, int32_t block_dim, std::size_t dynamic_smem = 0) -> uint32_t {
inline auto get_blocks_per_sm(T kernel, int32_t block_dim, std::size_t dynamic_smem = 0) -> uint32_t {
int num_blocks_per_sm = 0;
CHECK_CUDA(cudaOccupancyMaxActiveBlocksPerMultiprocessor(&num_blocks_per_sm, kernel, block_dim, dynamic_smem));
return static_cast<uint32_t>(num_blocks_per_sm);
@@ -164,7 +188,7 @@ inline auto get_runtime_version() -> int {
// Return the maximum dynamic shared memory per block for the given kernel
template <typename T>
inline auto get_available_dynamic_smem_per_block(T&& kernel, int num_blocks, int block_size) -> std::size_t {
inline auto get_available_dynamic_smem_per_block(T kernel, int num_blocks, int block_size) -> std::size_t {
std::size_t smem_size;
CHECK_CUDA(cudaOccupancyAvailableDynamicSMemPerBlock(&smem_size, kernel, num_blocks, block_size));
return smem_size;
@@ -190,6 +214,110 @@ inline auto get_max_smem_per_block(int device_id, bool opt_in = true, bool use_c
});
}
struct L1Carveout {
int carveout_percent;
uint32_t blocks_per_sm;
};
template <typename T>
inline void set_smem_carveout(T kernel, int percent) {
#ifdef USE_ROCM
(void)kernel, (void)percent;
#else
CHECK_CUDA(::cudaFuncSetAttribute(kernel, cudaFuncAttributePreferredSharedMemoryCarveout, percent));
#endif
}
template <typename T>
inline auto set_prefer_l1_carveout_with_occupancy(
const int lower_bound_percent,
const int upper_bound_percent,
const uint32_t min_occupancy,
T kernel,
const uint32_t block_threads,
const std::size_t dyn_smem_bytes = 0) -> L1Carveout {
#ifdef USE_ROCM
(void)lower_bound_percent, (void)upper_bound_percent, (void)min_occupancy;
return {-1, get_blocks_per_sm(kernel, block_threads, dyn_smem_bytes)};
#else
const auto kernel_ptr = reinterpret_cast<const void*>(kernel);
// The attribute and the occupancy query both act on the current device.
static_assert(cudaSharedmemCarveoutMaxL1 <= cudaSharedmemCarveoutMaxShared);
CHECK_HOST(
min_occupancy > 0 && lower_bound_percent <= upper_bound_percent //
&& lower_bound_percent >= cudaSharedmemCarveoutMaxL1 //
&& upper_bound_percent <= cudaSharedmemCarveoutMaxShared);
for (int p = lower_bound_percent; p <= upper_bound_percent; ++p) {
set_smem_carveout(kernel_ptr, p);
const auto occupancy = get_blocks_per_sm(kernel_ptr, block_threads, dyn_smem_bytes);
if (occupancy >= min_occupancy) return {p, occupancy};
}
SGL_PANIC << "no carveout in [" << lower_bound_percent << ", " << upper_bound_percent
<< "] can satisfy target occupancy " << min_occupancy << " for kernel " << kernel_ptr;
__builtin_unreachable();
#endif
}
/// \brief Prefer the largest L1 carveout that keeps `kernel`'s default occupancy on the current
/// device (PDL secondaries inherit the primary's carveout). The attribute is per device and sticky
/// for the process; panics when no carveout restores occupancy. ROCm has no such attribute and
/// reports percent -1. Wrap it in `init_per_device(device, ...)` to run it once per device.
template <typename T>
inline auto set_prefer_l1_carveout(T kernel, uint32_t block_threads, std::size_t dyn_smem_bytes = 0) -> L1Carveout {
#ifdef USE_ROCM
return {-1, get_blocks_per_sm(kernel, block_threads, dyn_smem_bytes)};
#else
const auto kernel_ptr = reinterpret_cast<const void*>(kernel);
set_smem_carveout(kernel_ptr, cudaSharedmemCarveoutDefault);
const auto default_occupancy = get_blocks_per_sm(kernel_ptr, block_threads, dyn_smem_bytes);
return set_prefer_l1_carveout_with_occupancy(
cudaSharedmemCarveoutMaxL1,
cudaSharedmemCarveoutMaxShared,
default_occupancy,
kernel_ptr,
block_threads,
dyn_smem_bytes);
#endif
}
/// \brief Opt `kernel` into `dyn_smem_bytes` of dynamic shared memory on the current device.
template <typename T>
inline void set_max_dynamic_smem(T kernel, std::size_t dyn_smem_bytes) {
const auto kernel_ptr = reinterpret_cast<const void*>(kernel);
const auto num_bytes = static_cast<int>(dyn_smem_bytes);
CHECK_CUDA(::cudaFuncSetAttribute(kernel_ptr, cudaFuncAttributeMaxDynamicSharedMemorySize, num_bytes));
}
/// \brief Run `fn` on the first call per device, keyed off the uniqueness of (`Tag`, `Fn`).
/// \note Relaxed on purpose, to keep the hot path a single atomic load: concurrent first callers
/// may all run `fn`, so it must be idempotent (e.g. setting a function attribute). A throwing `fn`
/// leaves the device uninitialized, and devices past `kNumStaticMaxDevice` rerun `fn` every call.
template <typename Tag = void, std::invocable Fn>
inline void init_per_device(int device_id, bool use_cache, Fn&& fn) {
static_assert(kNumStaticMaxDevice % 8 == 0);
constexpr uint32_t kNumBytes = kNumStaticMaxDevice / 8;
static std::atomic_uint8_t s_initialized[kNumBytes]{};
const auto device = static_cast<uint32_t>(device_id);
const auto can_use_cache = use_cache && device < kNumStaticMaxDevice;
const auto mask = static_cast<uint8_t>(1u << (device % 8));
if (can_use_cache && s_initialized[device / 8].load() & mask) return;
std::forward<Fn>(fn)();
if (can_use_cache) s_initialized[device / 8].fetch_or(mask);
}
// Some wrappers of the above functions
template <typename Tag = void, std::invocable Fn>
inline void init_per_device(details::MaybeDevice device, Fn&& fn) {
return init_per_device<Tag>(device.device_id, true, std::forward<Fn>(fn));
}
template <auto kernel>
inline void set_max_dynamic_smem_per_device(details::MaybeDevice device, std::size_t dyn_smem_bytes) {
using Tag = details::Tag<kernel>;
return init_per_device<Tag>(device.device_id, true, [=] { set_max_dynamic_smem(kernel, dyn_smem_bytes); });
}
} // namespace host::runtime
} // namespace sglang
@@ -11,7 +11,6 @@
/// - Typed `load_as` / `store_as` for void-pointer access.
/// - `pointer::offset` for safe void-pointer arithmetic.
/// - `host::LaunchKernel` - kernel launcher with optional PDL.
/// - `host::prefer_l1_carveout` / `host::ensure_prefer_l1` - occupancy-preserving L1 carveout preference.
/// - `host::RuntimeDeviceCheck` - CUDA error checking.
#pragma once
@@ -25,14 +24,10 @@
#include <concepts>
#include <cstddef>
#include <cstdint>
#include <map>
#include <mutex>
#include <optional>
#include <type_traits>
#include <utility>
#ifndef USE_ROCM
#include <tvm/ffi/extra/cuda/device_guard.h>
#include <cuda.h>
#include <cuda_bf16.h>
#include <cuda_fp16.h>
#include <cuda_fp8.h>
@@ -59,10 +54,6 @@ inline constexpr auto cudaSuccess = hipSuccess;
#define cudaDeviceGetAttribute hipDeviceGetAttribute
#define cudaDevAttrComputeCapabilityMajor hipDeviceAttributeComputeCapabilityMajor
#define cudaDevAttrComputeCapabilityMinor hipDeviceAttributeComputeCapabilityMinor
#define cudaGetDevice hipGetDevice
#ifndef cudaOccupancyMaxActiveBlocksPerMultiprocessor
#define cudaOccupancyMaxActiveBlocksPerMultiprocessor hipOccupancyMaxActiveBlocksPerMultiprocessor
#endif
#define cudaFuncSetAttribute hipFuncSetAttribute
#define cudaFuncAttributeMaxDynamicSharedMemorySize hipFuncAttributeMaxDynamicSharedMemorySize
#endif
@@ -337,13 +328,42 @@ SGL_DEVICE_HOST constexpr uint32_t get_tmem_cols(uint32_t num_cols) {
namespace host {
namespace runtime {
inline bool is_cuda_error(::cudaError_t error) {
return error != ::cudaSuccess;
}
inline const char* get_cuda_error_string(::cudaError_t error) {
return ::cudaGetErrorString(error);
}
#ifndef USE_ROCM
inline bool is_cuda_error(::CUresult error) {
return error != ::CUDA_SUCCESS;
}
inline const char* get_cuda_error_string(::CUresult error) {
const char* error_string = "";
// cuGetErrorString only fails with CUDA_ERROR_INVALID_VALUE, leaving the string null.
const auto new_error = ::cuGetErrorString(error, &error_string);
return new_error == ::CUDA_SUCCESS ? error_string : "Unknown error";
}
#endif
} // namespace runtime
/**
* \brief Check the CUDA error code and panic with location info on failure.
* \brief Check a CUDA runtime (`cudaError_t`) or driver (`CUresult`) error code
* and panic with location info on failure.
*/
inline void RuntimeDeviceCheck(::cudaError_t error, DebugInfo location = {}) {
if (error != ::cudaSuccess) {
template <typename T>
inline void RuntimeDeviceCheck(T error, DebugInfo location = {}) {
if (runtime::is_cuda_error(error)) {
[[unlikely]];
host::panic(location, "CUDA error: ", ::cudaGetErrorString(error));
panic(location, "CUDA error: ", runtime::get_cuda_error_string(error));
}
}
@@ -352,58 +372,6 @@ inline void RuntimeDeviceCheck(DebugInfo location = {}) {
return RuntimeDeviceCheck(::cudaGetLastError(), location);
}
/// \brief Where `prefer_l1_carveout` settled: the carveout percentage and the kernel's blocks/SM there.
struct L1Carveout {
int carveout_pct;
uint32_t blocks_per_sm;
};
/// \brief Prefer the largest L1 carveout that keeps `kernel`'s default occupancy on `device_id`
/// (PDL secondaries inherit the primary's carveout). Per device, sticky for the process; panics
/// when no carveout restores occupancy. ROCm has no such attribute and reports pct -1.
template <typename T>
inline auto prefer_l1_carveout(T&& kernel, int device_id, uint32_t block_threads, std::size_t dyn_smem_bytes = 0)
-> L1Carveout {
const auto blocks_per_sm = [&] {
int blocks = 0;
RuntimeDeviceCheck(::cudaOccupancyMaxActiveBlocksPerMultiprocessor(&blocks, kernel, block_threads, dyn_smem_bytes));
return static_cast<uint32_t>(blocks);
};
#ifdef USE_ROCM
(void)device_id;
return {-1, blocks_per_sm()};
#else
// The attribute and the occupancy query both act on the current device.
tvm::ffi::CUDADeviceGuard guard(device_id);
const auto set_carveout = [&](int pct) {
RuntimeDeviceCheck(::cudaFuncSetAttribute(kernel, cudaFuncAttributePreferredSharedMemoryCarveout, pct));
};
set_carveout(cudaSharedmemCarveoutDefault);
const uint32_t occupancy = blocks_per_sm();
RuntimeCheck(occupancy > 0, "kernel does not fit on an SM");
for (int pct = cudaSharedmemCarveoutMaxL1;; ++pct) {
set_carveout(pct);
if (const uint32_t now = blocks_per_sm(); now >= occupancy) return {pct, now};
RuntimeCheck(pct < cudaSharedmemCarveoutMaxShared, "no carveout restores occupancy ", occupancy);
}
#endif
}
/// \brief `prefer_l1_carveout` memoized per (kernel, device): the attribute is sticky, so a hit returns the
/// settled value without touching the device. Load-time configure() and the launch builder share the memo.
/// Callers must keep the block size and dynamic shared memory fixed for each (kernel, device), and
/// must not change the kernel's carveout attribute after configuration.
template <typename T>
inline auto ensure_prefer_l1(T kernel, int device_id, uint32_t block_threads, std::size_t dyn_smem_bytes = 0)
-> L1Carveout {
static std::mutex mutex;
static std::map<std::pair<const void*, int>, L1Carveout> settled;
const std::lock_guard<std::mutex> lock(mutex);
const auto key = std::make_pair(reinterpret_cast<const void*>(kernel), device_id);
if (const auto it = settled.find(key); it != settled.end()) return it->second;
return settled.emplace(key, prefer_l1_carveout(kernel, device_id, block_threads, dyn_smem_bytes)).first->second;
}
/**
* \brief Kernel launcher with automatic stream resolution and PDL support.
*
@@ -413,8 +381,6 @@ inline auto ensure_prefer_l1(T kernel, int device_id, uint32_t block_threads, st
* .enable_pdl(true)(my_kernel, arg0, arg1);
* host::LaunchKernel(grid, block, stream)
* .config({.use_pdl = true, .cluster_dim = cluster_dim})(my_kernel, arg0);
* host::LaunchKernel(grid, block, device)
* .config({.use_pdl = true, .prefer_l1 = true})(my_kernel, arg0);
* \endcode
*
* The constructor resolves the CUDA stream from a `DLDevice` (via `TVMFFIEnvGetStream`)
@@ -425,7 +391,6 @@ struct LaunchKernel {
struct KernelConfig {
bool use_pdl = false;
std::optional<dim3> cluster_dim = std::nullopt;
bool prefer_l1 = false; // `ensure_prefer_l1` on the launch device before the first launch
};
public:
@@ -436,8 +401,7 @@ struct LaunchKernel {
std::size_t dynamic_shared_mem_bytes = 0,
DebugInfo location = {}) noexcept
: m_config(s_make_config(grid_dim, block_dim, resolve_device(device), dynamic_shared_mem_bytes)),
m_location(location),
m_device_id(device.device_id) {}
m_location(location) {}
explicit LaunchKernel(
dim3 grid_dim,
@@ -445,9 +409,7 @@ struct LaunchKernel {
cudaStream_t stream,
std::size_t dynamic_shared_mem_bytes = 0,
DebugInfo location = {}) noexcept
: m_config(s_make_config(grid_dim, block_dim, stream, dynamic_shared_mem_bytes)),
m_location(location),
m_device_id(-1) {}
: m_config(s_make_config(grid_dim, block_dim, stream, dynamic_shared_mem_bytes)), m_location(location) {}
LaunchKernel(const LaunchKernel&) = delete;
LaunchKernel& operator=(const LaunchKernel&) = delete;
@@ -483,11 +445,6 @@ struct LaunchKernel {
return *this;
}
auto prefer_l1(bool enabled = true) -> LaunchKernel& {
m_prefer_l1 = enabled;
return *this;
}
/**
* \brief Configure the kernel launch with the given options.
* \param config The kernel configuration options.
@@ -499,13 +456,11 @@ struct LaunchKernel {
auto config(const KernelConfig& config) -> LaunchKernel& {
if (config.use_pdl) this->enable_pdl(true);
if (config.cluster_dim) this->enable_cluster(*config.cluster_dim);
if (config.prefer_l1) this->prefer_l1(true);
return *this;
}
template <typename T, typename... Args>
auto operator()(T&& kernel, Args&&... args) const -> void {
if (m_prefer_l1) apply_prefer_l1(kernel);
#ifdef USE_ROCM
hipLaunchKernelGGL(
std::forward<T>(kernel),
@@ -540,28 +495,15 @@ struct LaunchKernel {
return config;
}
// Memo hit after load-time configure(); stream-constructed launches use the current device.
template <typename T>
void apply_prefer_l1(T&& kernel) const {
int device_id = m_device_id;
if (device_id < 0) RuntimeDeviceCheck(::cudaGetDevice(&device_id));
const dim3 block = m_config.blockDim;
ensure_prefer_l1(+kernel, device_id, block.x * block.y * block.z, m_config.dynamicSmemBytes);
}
cudaLaunchConfig_t m_config;
const DebugInfo m_location;
cudaLaunchAttribute m_attrs[2];
int m_device_id;
bool m_prefer_l1 = false;
};
// The empty-true-branch if/else form keeps a trailing `else` in user code
// bound to the user's `if`, not to the macro's.
#define CHECK_CUDA(COND) \
if (const auto error = (COND); error == ::cudaSuccess) [[likely]] { \
} else \
host::Error() << "CUDA error: " << ::cudaGetErrorString(error) << ". "
#define CHECK_CUDA(COND) \
if (const auto _err = (COND); !::sglang::host::runtime::is_cuda_error(_err)) [[likely]] { \
} else \
SGL_PANIC << "CUDA error: " << ::sglang::host::runtime::get_cuda_error_string(_err) << ". "
} // namespace host
@@ -178,37 +178,28 @@ inline auto irange(T start, T end) {
/** \brief Error class for stream-style error logging. */
struct Error {
Error(DebugInfo location = {}) {
m_oss << "Failed at " << location.file_name() << ":" << location.line() << ": ";
std::ostringstream stream;
explicit Error(DebugInfo location = {}) {
stream << "Failed at " << location.file_name() << ":" << location.line() << ": ";
}
template <typename T>
Error& operator<<(T&& arg) {
m_oss << std::forward<T>(arg);
return *this;
}
[[noreturn]]
~Error() noexcept(false) {
throw PanicError(std::move(m_oss).str());
throw PanicError(std::move(stream).str());
}
private:
std::ostringstream m_oss;
};
#define SGL_PANIC ::sglang::host::Error().stream
/**
* \brief 0-overhead CHECK macro for host code. This can avoid unnecessary
* instantiation of error messages when the condition is true.
*
* Usage: CHECK_HOST(ptr != nullptr) << "Pointer must not be null";
*/
// The empty-true-branch if/else form keeps a trailing `else` in user code
// bound to the user's `if`, not to the macro's.
#define CHECK_HOST(COND) \
if (COND) [[likely]] { \
} else \
host::Error()
SGL_PANIC
} // namespace host
+2
View File
@@ -353,6 +353,8 @@ class Envs:
SGLANG_LOG_REQUEST_HEADERS = EnvTuple(tuple())
SGLANG_LOG_SCHEDULER_STATUS_TARGET = EnvStr("")
SGLANG_LOG_SCHEDULER_STATUS_INTERVAL = EnvFloat(60.0)
# Read by JIT kernel headers (sgl_kernel/logging.h): DEBUG | INFO | WARNING | ERROR.
SGLANG_JIT_LOG_LEVEL = EnvStr("WARNING")
SGLANG_ENABLE_RANK_CONSENSUS_CHECKER = EnvBool(False)
# ===================================================================