From 4f5fcbfef586748394c0086d22014ca5272fb88f Mon Sep 17 00:00:00 2001 From: Ziyi Xu <76582120+DarkSharpness@users.noreply.github.com> Date: Wed, 14 Oct 2026 10:46:51 +0800 Subject: [PATCH] [JIT] Clean up kernel headers: move runtime helpers, add once.h (#41147) Co-authored-by: Claude Opus 5.5 (1M context) --- .../kernels/jit/csrc/deepseek_v4/c_plan.cuh | 4 +- .../jit/csrc/deepseek_v4/topk_bf16_small.cuh | 8 +- .../kernels/jit/csrc/deepseek_v4/topk_v1.cuh | 17 +- .../kernels/jit/csrc/deepseek_v4/topk_v2.cuh | 9 +- .../jit/csrc/deepseek_v4/wo_a_fused.cuh | 10 +- .../jit/csrc/dsa/kpool_topk_transform.cuh | 18 +- .../gemm/convrot_int8/convrot_int8_entry.cuh | 1 + .../jit/csrc/gemm/dsv3_fused_a_gemm.cuh | 7 +- .../jit/csrc/kvcacheio/hicache_tma.cuh | 19 +- .../kernels/jit/csrc/kvcacheio/hisparse.cuh | 5 +- .../kernels/jit/csrc/speculative/sampling.cuh | 5 +- .../jit/include/sgl_kernel/allocator.h | 40 ----- .../kernels/jit/include/sgl_kernel/logging.h | 54 ++++++ .../kernels/jit/include/sgl_kernel/once.h | 147 +++++++++++++++ .../jit/include/sgl_kernel/runtime.cuh | 168 +++++++++++++++--- .../kernels/jit/include/sgl_kernel/utils.cuh | 138 +++++--------- .../kernels/jit/include/sgl_kernel/utils.h | 23 +-- python/sglang/srt/environ.py | 2 + .../ops/elementwise/launch_carveout.cuh | 64 ------- .../ops/elementwise/test_launch_carveout.py | 45 ----- 20 files changed, 436 insertions(+), 348 deletions(-) delete mode 100644 python/sglang/kernels/jit/include/sgl_kernel/allocator.h create mode 100644 python/sglang/kernels/jit/include/sgl_kernel/logging.h create mode 100644 python/sglang/kernels/jit/include/sgl_kernel/once.h delete mode 100644 test/registered/kernels/ops/elementwise/launch_carveout.cuh delete mode 100644 test/registered/kernels/ops/elementwise/test_launch_carveout.py diff --git a/python/sglang/kernels/jit/csrc/deepseek_v4/c_plan.cuh b/python/sglang/kernels/jit/csrc/deepseek_v4/c_plan.cuh index 2d1419a942..a797820c9f 100644 --- a/python/sglang/kernels/jit/csrc/deepseek_v4/c_plan.cuh +++ b/python/sglang/kernels/jit/csrc/deepseek_v4/c_plan.cuh @@ -29,12 +29,12 @@ using F2S_T = int64_t; using IDX_T = int64_t; /// NOTE: for the internal use, we pack the ragged and batch id, since both not exceed 65536 -SGL_DEVICE __host__ PlanW pack_w(uint32_t ragged_id, uint32_t batch_id, int32_t seq_len) { +SGL_DEVICE_HOST PlanW pack_w(uint32_t ragged_id, uint32_t batch_id, int32_t seq_len) { return {static_cast(ragged_id | batch_id << 16), seq_len}; } /// NOTE: for the internal use, we pack the ragged and batch id, since both not exceed 65536 -SGL_DEVICE uint2 unpack_w(PlanW plan) { +SGL_DEVICE_HOST uint2 unpack_w(PlanW plan) { return {static_cast(plan.ragged_id), static_cast(plan.ragged_id >> 16)}; } diff --git a/python/sglang/kernels/jit/csrc/deepseek_v4/topk_bf16_small.cuh b/python/sglang/kernels/jit/csrc/deepseek_v4/topk_bf16_small.cuh index cec06c8541..70aad691f8 100644 --- a/python/sglang/kernels/jit/csrc/deepseek_v4/topk_bf16_small.cuh +++ b/python/sglang/kernels/jit/csrc/deepseek_v4/topk_bf16_small.cuh @@ -85,12 +85,6 @@ struct TopKBF16Params { uint32_t page_bits; }; -SGL_DEVICE uint32_t get_ptx_lane_id() { - uint32_t lane_id; - asm volatile("mov.u32 %0, %%laneid;" : "=r"(lane_id)); - return lane_id; -} - /// \brief Exclusive suffix scan: lane `L` gets the sum over lanes `> L`. SGL_DEVICE uint32_t warp_exclusive_suffix_sum(uint32_t x, uint32_t lane_id) { uint32_t inc = x; @@ -183,7 +177,7 @@ __global__ __launch_bounds__(TopKBF16Config::kBlockSize, TopKBF16Config::kOccupa const auto bx = blockIdx.x; const auto tx = threadIdx.x; - const auto lane_id = get_ptx_lane_id(); + const auto lane_id = warp::get_lane_id(); const auto warp_id = tx / kWarpThreads; const auto topk = params.topk; // a selected index i maps through this row's table to slot diff --git a/python/sglang/kernels/jit/csrc/deepseek_v4/topk_v1.cuh b/python/sglang/kernels/jit/csrc/deepseek_v4/topk_v1.cuh index 121a31e673..da3bbaaa4a 100644 --- a/python/sglang/kernels/jit/csrc/deepseek_v4/topk_v1.cuh +++ b/python/sglang/kernels/jit/csrc/deepseek_v4/topk_v1.cuh @@ -1,6 +1,7 @@ #include #include +#include #include #include @@ -14,8 +15,8 @@ namespace sglang { // `topk` is a *runtime* value (<= kMaxTopK), so one module serves every k. It // used to be baked in via -DSGL_TOPK, which built a separate module per k -- // and because `kTopK` came from a macro rather than a template parameter, both -// modules exported identically mangled symbols. The function-local static in -// setup_kernel_smem_once() is emitted as STB_GNU_UNIQUE, which the loader +// modules exported identically mangled symbols. The function-local static that +// guarded the smem opt-in is emitted as STB_GNU_UNIQUE, which the loader // merges across every loaded object, so whichever module was used second // skipped its cudaFuncSetAttribute opt-in and then failed to launch with 64 KB // of dynamic shared memory ("invalid argument"). @@ -270,16 +271,6 @@ __global__ void topk_transform_kernel(const __grid_constant__ TopKParams params) device::PDLTriggerSecondary(); } -template -void setup_kernel_smem_once(host::DebugInfo where = {}) { - [[maybe_unused]] - static const auto result = [] { - const auto fptr = std::bit_cast(f); - return ::cudaFuncSetAttribute(fptr, ::cudaFuncAttributeMaxDynamicSharedMemorySize, kMaxDynamicSMEM); - }(); - host::RuntimeDeviceCheck(result, where); -} - template struct TopKKernel { static constexpr auto kernel = topk_transform_kernel; @@ -344,7 +335,7 @@ struct TopKKernel { .topk = topk, }; constexpr auto kSMEM_ = kSMEM + sizeof(int32_t); // align up a little - setup_kernel_smem_once(); + runtime::set_max_dynamic_smem_per_device(device.unwrap(), kSMEM_); LaunchKernel(batch_size, kTopKBlockSize, device.unwrap(), kSMEM_).enable_pdl(kUsePDL)(kernel, params); } }; diff --git a/python/sglang/kernels/jit/csrc/deepseek_v4/topk_v2.cuh b/python/sglang/kernels/jit/csrc/deepseek_v4/topk_v2.cuh index fae0ece4c6..4272353273 100644 --- a/python/sglang/kernels/jit/csrc/deepseek_v4/topk_v2.cuh +++ b/python/sglang/kernels/jit/csrc/deepseek_v4/topk_v2.cuh @@ -12,6 +12,7 @@ #include #include +#include #include #include @@ -1103,12 +1104,10 @@ struct TopKKernel { constexpr uint32_t kClusterSize = 16; // Widths above 8 are non-portable; the launch is rejected without this. const auto kernel = topk_small_batch_cluster_kernel; - [[maybe_unused]] - static const bool _ = [&kernel] { - const auto kernel_ptr = reinterpret_cast(kernel); + const auto kernel_ptr = reinterpret_cast(kernel); + runtime::init_per_device(device, [kernel_ptr] { CHECK_CUDA(::cudaFuncSetAttribute(kernel_ptr, ::cudaFuncAttributeNonPortableClusterSizeAllowed, 1)); - return true; - }(); + }); return LaunchKernel({batch_size, kClusterSize}, kBlockSize, device) .config({.use_pdl = kUsePDL, .cluster_dim = dim3{1, kClusterSize}}) .launch(kernel, params); diff --git a/python/sglang/kernels/jit/csrc/deepseek_v4/wo_a_fused.cuh b/python/sglang/kernels/jit/csrc/deepseek_v4/wo_a_fused.cuh index 3bcd3b13b8..6acbca0802 100644 --- a/python/sglang/kernels/jit/csrc/deepseek_v4/wo_a_fused.cuh +++ b/python/sglang/kernels/jit/csrc/deepseek_v4/wo_a_fused.cuh @@ -22,6 +22,7 @@ #include #include +#include #include #include #include @@ -779,11 +780,10 @@ inline void wo_a_fused_run( const auto launch = [&](const auto* pos) { using PosT = std::decay_t; constexpr auto kernel = wo_a_fused_kernel; - [[maybe_unused]] static const auto _ = [] { - CHECK_CUDA(cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, sizeof(Smem))); - CHECK_CUDA(cudaFuncSetAttribute(kernel, cudaFuncAttributePreferredSharedMemoryCarveout, 100)); - return 0; - }(); + runtime::init_per_device(device, [=] { + runtime::set_max_dynamic_smem(kernel, sizeof(Smem)); + runtime::set_smem_carveout(kernel, cudaSharedmemCarveoutMaxShared); + }); LaunchKernel(kClusters * kSplitK, kThreads, device, sizeof(Smem)) .config({.use_pdl = true, .cluster_dim = dim3{kSplitK, 1, 1}})( kernel, diff --git a/python/sglang/kernels/jit/csrc/dsa/kpool_topk_transform.cuh b/python/sglang/kernels/jit/csrc/dsa/kpool_topk_transform.cuh index 18632124eb..0d912af91c 100644 --- a/python/sglang/kernels/jit/csrc/dsa/kpool_topk_transform.cuh +++ b/python/sglang/kernels/jit/csrc/dsa/kpool_topk_transform.cuh @@ -3,12 +3,12 @@ #include #include +#include #include #include #include -#include #include #include #ifndef USE_ROCM @@ -354,20 +354,6 @@ __global__ __launch_bounds__(kThreadsPerBlock) void kpool_topk_transform_kernel( } } -template -void setup_kernel_smem_once(host::DebugInfo where = {}) { - [[maybe_unused]] - static const auto result = [] { - const auto fptr = std::bit_cast(f); -#if defined(__HIP_PLATFORM_AMD__) - return ::hipFuncSetAttribute(fptr, ::hipFuncAttributeMaxDynamicSharedMemorySize, kMaxDynamicSMEM); -#else - return ::cudaFuncSetAttribute(fptr, ::cudaFuncAttributeMaxDynamicSharedMemorySize, kMaxDynamicSMEM); -#endif - }(); - host::RuntimeDeviceCheck(result, where); -} - template const T* optional_data_ptr(const tvm::ffi::Optional& opt) { if (!opt.has_value()) { @@ -463,7 +449,7 @@ struct KpoolTopKTransformKernel { .input_stride = static_cast(S.unwrap()), }; - setup_kernel_smem_once(); + runtime::set_max_dynamic_smem_per_device(device.unwrap().device_id, kSmem); LaunchKernel(batch_size, kThreadsPerBlock, device.unwrap(), kSmem)( kernel, params, diff --git a/python/sglang/kernels/jit/csrc/gemm/convrot_int8/convrot_int8_entry.cuh b/python/sglang/kernels/jit/csrc/gemm/convrot_int8/convrot_int8_entry.cuh index 3c11ba2d7b..b13b5ac234 100644 --- a/python/sglang/kernels/jit/csrc/gemm/convrot_int8/convrot_int8_entry.cuh +++ b/python/sglang/kernels/jit/csrc/gemm/convrot_int8/convrot_int8_entry.cuh @@ -48,6 +48,7 @@ limitations under the License. #include #include #include +#include #include "cutlass_extensions/epilogue/epilogue_per_row_per_col_scale.h" #include "cutlass_extensions/gemm/gemm_universal_base_compat.h" diff --git a/python/sglang/kernels/jit/csrc/gemm/dsv3_fused_a_gemm.cuh b/python/sglang/kernels/jit/csrc/gemm/dsv3_fused_a_gemm.cuh index 509113b4b2..a2c9edb304 100644 --- a/python/sglang/kernels/jit/csrc/gemm/dsv3_fused_a_gemm.cuh +++ b/python/sglang/kernels/jit/csrc/gemm/dsv3_fused_a_gemm.cuh @@ -21,6 +21,7 @@ #include #include +#include #include #include @@ -593,9 +594,9 @@ void invokeFusedAGemm(T* output, T const* mat_a, T const* mat_b, int num_tokens, dim3 grid(cta_m_cnt, cta_n_cnt, 1); dim3 block_size(256); - auto kernel = fused_a_gemm_kernel; - if (smem_bytes >= (48 * 1024)) { - host::RuntimeDeviceCheck(cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, smem_bytes)); + constexpr auto kernel = fused_a_gemm_kernel; + if constexpr (smem_bytes >= (48 * 1024)) { + host::runtime::set_max_dynamic_smem_per_device(device.device_id, smem_bytes); } host::LaunchKernel(grid, block_size, device, smem_bytes).enable_pdl(kUsePDL)(kernel, output, mat_a, mat_b, gemm_n); } diff --git a/python/sglang/kernels/jit/csrc/kvcacheio/hicache_tma.cuh b/python/sglang/kernels/jit/csrc/kvcacheio/hicache_tma.cuh index 1dda34cda4..c3dc8991b0 100644 --- a/python/sglang/kernels/jit/csrc/kvcacheio/hicache_tma.cuh +++ b/python/sglang/kernels/jit/csrc/kvcacheio/hicache_tma.cuh @@ -28,6 +28,7 @@ #include #include +#include #include #include @@ -130,7 +131,7 @@ inline constexpr uint32_t kHicacheTmaMaxMapRowBytes = 256 * 8; // Rows per chunk: largest power of two that fits the stage, so chunks never // straddle a (power-of-two) page and a page run stays one bulk op. -__host__ __device__ constexpr uint32_t hicache_tma_rows_per_chunk(uint32_t stage_bytes, uint32_t row_bytes) { +SGL_DEVICE_HOST constexpr uint32_t hicache_tma_rows_per_chunk(uint32_t stage_bytes, uint32_t row_bytes) { uint32_t rows = 1; while (rows * 2 <= stage_bytes / row_bytes && rows * 2 <= kHicacheTmaMaxRows) rows *= 2; @@ -438,16 +439,12 @@ struct HiCacheTmaKernel { (params.has_v ? 2u : 1u) * params.num_layers * div_ceil(params.length, rows_per_chunk(params.row_bytes)); constexpr std::size_t kSmemBytes = sizeof(Smem); - static const bool attr_set = [] { - for (auto fn : {kernel, kernel}) { - RuntimeDeviceCheck( - cudaFuncSetAttribute(fn, cudaFuncAttributeMaxDynamicSharedMemorySize, static_cast(kSmemBytes))); - } - return true; - }(); - (void)attr_set; - LaunchKernel(std::min(chunks, kBlockQuota), kThreads, device, kSmemBytes)( - use_int32 ? kernel : kernel, params); + runtime::init_per_device(device, [=] { + runtime::set_max_dynamic_smem(kernel, kSmemBytes); + runtime::set_max_dynamic_smem(kernel, kSmemBytes); + }); + const auto fn = use_int32 ? kernel : kernel; + LaunchKernel(std::min(chunks, kBlockQuota), kThreads, device, kSmemBytes)(fn, params); } // Cache operand viewed as [-1, D] rows; binds row dim, stride and dtype. diff --git a/python/sglang/kernels/jit/csrc/kvcacheio/hisparse.cuh b/python/sglang/kernels/jit/csrc/kvcacheio/hisparse.cuh index 9c530fa62b..bf2b62ba47 100644 --- a/python/sglang/kernels/jit/csrc/kvcacheio/hisparse.cuh +++ b/python/sglang/kernels/jit/csrc/kvcacheio/hisparse.cuh @@ -779,7 +779,10 @@ void load_cache_to_device_buffer( constexpr size_t smem_bytes = SmemLayout::BYTES; #ifndef USE_ROCM if constexpr (smem_bytes > 48u * 1024u) { - cudaFuncSetAttribute(kernel_fn, cudaFuncAttributeMaxDynamicSharedMemorySize, smem_bytes); + /// NOTE: for each launch template, the `kernel_fn` must be unique + runtime::init_per_device(kernel_device, [=] { // + runtime::set_max_dynamic_smem(kernel_fn, smem_bytes); + }); } #endif LaunchKernel(bs, BLOCK_SIZE, device, smem_bytes)( diff --git a/python/sglang/kernels/jit/csrc/speculative/sampling.cuh b/python/sglang/kernels/jit/csrc/speculative/sampling.cuh index 90c17c6b4a..04f449100f 100644 --- a/python/sglang/kernels/jit/csrc/speculative/sampling.cuh +++ b/python/sglang/kernels/jit/csrc/speculative/sampling.cuh @@ -18,6 +18,7 @@ #include +#include #include #include @@ -78,7 +79,7 @@ void tree_speculative_sampling_target_only( constexpr uint32_t block_threads = 1024; constexpr size_t smem_size = sizeof(SamplingTempStorage); - auto kernel = speculative_sampling::TreeSpeculativeSamplingTargetOnly< + constexpr auto kernel = speculative_sampling::TreeSpeculativeSamplingTargetOnly< block_threads, SCAN_ALGO, REDUCE_ALGO, @@ -87,7 +88,7 @@ void tree_speculative_sampling_target_only( float, int32_t, int64_t>; - CHECK_CUDA(cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, smem_size)); + runtime::set_max_dynamic_smem_per_device(device.unwrap().device_id, smem_size); LaunchKernel(static_cast(batch_size.unwrap()), block_threads, device.unwrap(), smem_size)( kernel, static_cast(predicts.data_ptr()), diff --git a/python/sglang/kernels/jit/include/sgl_kernel/allocator.h b/python/sglang/kernels/jit/include/sgl_kernel/allocator.h deleted file mode 100644 index 9b034f59a7..0000000000 --- a/python/sglang/kernels/jit/include/sgl_kernel/allocator.h +++ /dev/null @@ -1,40 +0,0 @@ -#pragma once - -#include -#include -#include -#include - -namespace sglang::host { - -/** - * \brief Allocate only once for a given function. - * \tparam kThreadSafe Whether to make the allocation thread-safe. - * \tparam Salt A salt type to avoid cache collision. - * \param key The key to identify the allocation. It should be unique for each allocation. - * \param callback The callback function to perform the allocation. It should return the allocated value. - * \note The `Fn` type must be unique. It's typically a lambda type that's evaluated only once. - * Otherwise, different call-sites may hit the same cache entry. - * In case where `Fn` is not unique (e.g. std::function), make `Salt` unique to avoid cache collision. - */ -template -inline auto allocate_once(Key&& key, Fn&& callback) -> std::decay_t>& { - using Value = std::decay_t>; - static std::unordered_map, Value> s_map; - const auto alloc = [&]() -> Value& { - const auto iter = s_map.find(key); - if (iter != s_map.end()) return iter->second; - // Evaluate the callback before inserting, so a throwing callback leaves no empty entry behind. - auto value = std::forward(callback)(); - return s_map.emplace(std::forward(key), std::move(value)).first->second; - }; - if constexpr (kThreadSafe) { - static std::mutex s_mutex; - const auto lock = std::lock_guard{s_mutex}; - return alloc(); - } else { - return alloc(); - } -} - -} // namespace sglang::host diff --git a/python/sglang/kernels/jit/include/sgl_kernel/logging.h b/python/sglang/kernels/jit/include/sgl_kernel/logging.h new file mode 100644 index 0000000000..72ccc0b57b --- /dev/null +++ b/python/sglang/kernels/jit/include/sgl_kernel/logging.h @@ -0,0 +1,54 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include + +namespace sglang::host { + +enum class LogLevel : uint8_t { + DEBUG_ = 0, + INFO_ = 1, + WARNING_ = 2, + ERROR_ = 3, + DEFAULT = WARNING_, +}; + +inline const LogLevel kLogLevel = [] { + const auto env = std::getenv("SGLANG_JIT_LOG_LEVEL"); + if (!env) return LogLevel::DEFAULT; + const auto str = std::string_view{env}; + if (str == "DEBUG" || str == "debug") return LogLevel::DEBUG_; + if (str == "INFO" || str == "info") return LogLevel::INFO_; + if (str == "WARNING" || str == "warning") return LogLevel::WARNING_; + if (str == "ERROR" || str == "error") return LogLevel::ERROR_; + std::clog << "[WARNING] unrecognized SGLANG_JIT_LOG_LEVEL value '" << str << "'; using default log level." + << std::endl; + return LogLevel::DEFAULT; +}(); + +struct Logger { + std::ostringstream stream; + explicit Logger(LogLevel level, DebugInfo location = {}) { + static const std::string_view kPrefix[] = {"[DEBUG]", "[INFO]", "[WARNING]", "[ERROR]"}; + stream << kPrefix[static_cast(level)] << " [" << location.file_name() << ":" << location.line() << "] "; + } + ~Logger() { + std::clog << std::move(stream).str() << std::endl; + } +}; + +#define SGL_LOG(LEVEL) \ + if (const auto _log_level = ::sglang::host::LogLevel::LEVEL; _log_level < ::sglang::host::kLogLevel) { \ + } else \ + ::sglang::host::Logger(_log_level).stream + +#define SGL_LOG_DEBUG SGL_LOG(DEBUG_) +#define SGL_LOG_INFO SGL_LOG(INFO_) +#define SGL_LOG_WARNING SGL_LOG(WARNING_) + +} // namespace sglang::host diff --git a/python/sglang/kernels/jit/include/sgl_kernel/once.h b/python/sglang/kernels/jit/include/sgl_kernel/once.h new file mode 100644 index 0000000000..cc27bec968 --- /dev/null +++ b/python/sglang/kernels/jit/include/sgl_kernel/once.h @@ -0,0 +1,147 @@ +/// \file once.h +/// \brief Host-side "run it only once" helpers. +/// +/// C++ spells this `std::call_once` plus a caller-provided `std::once_flag`. The helpers here +/// hide the flag inside the function, keyed off the uniqueness of the callable's type, and hand +/// back the value the callable produced. They come in two forms: one value per call-site, or +/// one value per (call-site, key). + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace sglang::host { + +namespace details { + +/// \brief Fold any number of sub-hashes into one; the mixing step is boost's. +/// Defined for zero parts too, so an empty tuple key hashes without a special case. +inline std::size_t hash_combine(std::same_as auto... parts) { + std::size_t seed = 0; + ((seed ^= parts + 0x9e3779b97f4a7c15ULL + (seed << 6) + (seed >> 2)), ...); + return seed; +} + +/// \brief Hash for keys `std::hash` does not cover: pairs and tuples, recursively. +struct GenericHash { + template + std::size_t operator()(const T& value) const { + return std::hash{}(value); + } + template + std::size_t operator()(const std::pair& value) const { + return hash_combine((*this)(value.first), (*this)(value.second)); + } + template + std::size_t operator()(const std::tuple& value) const { + return std::apply([this](const auto&... args) { return hash_combine((*this)(args)...); }, value); + } +}; + +struct DummyMutex { + void lock() {} + void unlock() {} +}; + +} // namespace details + +template +struct CacheMap { + public: + CacheMap() = default; + CacheMap(const CacheMap&) = delete; + CacheMap& operator=(const CacheMap&) = delete; + + template + auto get_cached(KeyArg&& key, Fn&& factory) -> Value& { + const auto lock = std::lock_guard{m_mutex}; + if (const auto iter = m_map.find(key); iter != m_map.end()) return iter->second; + // Evaluate the factory before inserting, so a throwing factory leaves no entry behind. + auto value = std::forward(factory)(); + return m_map.try_emplace(std::forward(key), std::move(value)).first->second; + } + + private: + [[no_unique_address]] + std::conditional_t m_mutex; + std::unordered_map m_map; +}; + +template +struct CacheMap { + public: + CacheMap() = default; + CacheMap(const CacheMap&) = delete; + CacheMap& operator=(const CacheMap&) = delete; + + template + void get_cached(KeyArg&& key, Fn&& factory) { + const auto lock = std::lock_guard{m_mutex}; + if (m_set.contains(key)) return; + // Run the factory before inserting, so a throwing factory is retried on the next call. + std::forward(factory)(); + m_set.emplace(std::forward(key)); + } + + private: + [[no_unique_address]] + std::conditional_t m_mutex; + std::unordered_set m_set; +}; + +template +using CacheSet = CacheMap; + +/** + * \brief A keyed `std::call_once` that also hands back the value. + * Runs `factory` at most once per (call-site, key) and returns a stable reference to its result. + * \tparam Tag A tag type to avoid cache collision. + * \tparam kThreadSafe Whether to make the initialization thread-safe. Turn it off only where the + * call-site is provably single-threaded: the lock is taken on every call, cache hits included. + * \param key The key to identify the value. Each distinct key is initialized independently. + * \param factory The function to produce the value on first use. It should return the value. + * \note The `Fn` type must be unique. It's typically a lambda type that's evaluated only once. + * Otherwise, different call-sites may hit the same cache entry. + * In case where `Fn` is not unique (e.g. std::function), make `Tag` unique to avoid cache collision. + * \note `factory` runs with the lock held, so it must not re-enter this same instantiation. + */ +template +inline auto init_once(Key&& key, Fn&& factory) -> decltype(auto) { + static CacheMap, std::decay_t>, kThreadSafe> s_cache; + return s_cache.get_cached(std::forward(key), std::forward(factory)); +} + +/** + * \brief `std::call_once` without the caller-provided `once_flag`, also handing back the value. + * Runs `factory` at most once per call-site and returns a stable reference to its result. + * \tparam Tag A tag type to avoid cache collision. + * \param factory The function to produce the value on first use. It should return the value. + * \note The `Fn` type must be unique, for the same reason as the keyed overload above: the static + * lives in the instantiation, so a non-unique `Fn` (e.g. std::function) makes two call-sites share + * one value. Make `Tag` unique in that case. + * \note Always thread-safe, and deliberately has no `kThreadSafe` knob: C++ already guarantees that + * concurrent callers block until the first one finishes, and the post-initialization fast path is + * just a guard check, so there is no lock worth skipping. + * \note If `factory` throws, the value stays uninitialized and the next call retries, matching the + * keyed overload. `factory` must not re-enter this same instantiation; that is undefined behavior. + */ +template +inline auto init_once(Fn&& factory) -> decltype(auto) { + if constexpr (std::is_same_v, void>) { + static std::once_flag s_once; + return std::call_once(s_once, std::forward(factory)); + } else { + static auto result = std::forward(factory)(); + return (result); // parenthesized: `decltype(auto)` then deduces a reference, not a copy + } +} + +} // namespace sglang::host diff --git a/python/sglang/kernels/jit/include/sgl_kernel/runtime.cuh b/python/sglang/kernels/jit/include/sgl_kernel/runtime.cuh index 44e61ee505..77de08890f 100644 --- a/python/sglang/kernels/jit/include/sgl_kernel/runtime.cuh +++ b/python/sglang/kernels/jit/include/sgl_kernel/runtime.cuh @@ -6,13 +6,19 @@ #pragma once +#include + #include +#include #include +#include +#include #include #include #include + #ifndef USE_ROCM #include #else @@ -59,21 +65,22 @@ namespace sglang { namespace host::runtime { -inline void* get_device_accessible_ptr(const tvm::ffi::TensorView& tensor) { - void* ptr = tensor.data_ptr(); - const auto tensor_device_type = tensor.device().device_type; - if (tensor_device_type != kDLCPU && tensor_device_type != kDLGPUHost) { - return ptr; - } +namespace details { - void* device_ptr = nullptr; -#ifdef USE_ROCM - RuntimeDeviceCheck(::hipHostGetDevicePointer(&device_ptr, ptr, 0)); -#else - RuntimeDeviceCheck(::cudaHostGetDevicePointer(&device_ptr, ptr, 0)); -#endif - return device_ptr; -} +template +struct Tag {}; + +struct MaybeDevice { + MaybeDevice(DLDevice device) : device_id(device.device_id) {} + MaybeDevice(int32_t device_id_) : device_id(device_id_) {} + uint32_t device_id; +}; + +} // namespace details + +// Generous bound on the device ordinals one process can see; a larger ordinal +// is not an error, it just falls through to the driver query uncached. +inline constexpr uint32_t kNumStaticMaxDevice = 72; /** * \brief Memo for a per-device driver query, keyed by device ordinal. @@ -85,10 +92,6 @@ inline void* get_device_accessible_ptr(const tvm::ffi::TensorView& tensor) { template struct DeviceCacheMap { public: - // Generous bound on the device ordinals one process can see; a larger ordinal - // is not an error, it just falls through to the driver query uncached. - static constexpr uint32_t kNumStaticMaxDevice = 72; - constexpr DeviceCacheMap() { for (uint32_t i = 0; i < kNumStaticMaxDevice; ++i) { m_data[i] = kDefault; @@ -108,13 +111,34 @@ struct DeviceCacheMap { return value; } + template + T get_cached(int32_t device_, Fn&& fn) { + return this->get_cached(device_, true, std::forward(fn)); + } + private: T m_data[kNumStaticMaxDevice]; }; +inline void* get_device_accessible_ptr(const tvm::ffi::TensorView& tensor) { + void* ptr = tensor.data_ptr(); + const auto tensor_device_type = tensor.device().device_type; + if (tensor_device_type != kDLCPU && tensor_device_type != kDLGPUHost) { + return ptr; + } + + void* device_ptr = nullptr; +#ifdef USE_ROCM + CHECK_CUDA(::hipHostGetDevicePointer(&device_ptr, ptr, 0)); +#else + CHECK_CUDA(::cudaHostGetDevicePointer(&device_ptr, ptr, 0)); +#endif + return device_ptr; +} + // Return the maximum number of active blocks per SM for the given kernel template -inline auto get_blocks_per_sm(T&& kernel, int32_t block_dim, std::size_t dynamic_smem = 0) -> uint32_t { +inline auto get_blocks_per_sm(T kernel, int32_t block_dim, std::size_t dynamic_smem = 0) -> uint32_t { int num_blocks_per_sm = 0; CHECK_CUDA(cudaOccupancyMaxActiveBlocksPerMultiprocessor(&num_blocks_per_sm, kernel, block_dim, dynamic_smem)); return static_cast(num_blocks_per_sm); @@ -164,7 +188,7 @@ inline auto get_runtime_version() -> int { // Return the maximum dynamic shared memory per block for the given kernel template -inline auto get_available_dynamic_smem_per_block(T&& kernel, int num_blocks, int block_size) -> std::size_t { +inline auto get_available_dynamic_smem_per_block(T kernel, int num_blocks, int block_size) -> std::size_t { std::size_t smem_size; CHECK_CUDA(cudaOccupancyAvailableDynamicSMemPerBlock(&smem_size, kernel, num_blocks, block_size)); return smem_size; @@ -190,6 +214,110 @@ inline auto get_max_smem_per_block(int device_id, bool opt_in = true, bool use_c }); } +struct L1Carveout { + int carveout_percent; + uint32_t blocks_per_sm; +}; + +template +inline void set_smem_carveout(T kernel, int percent) { +#ifdef USE_ROCM + (void)kernel, (void)percent; +#else + CHECK_CUDA(::cudaFuncSetAttribute(kernel, cudaFuncAttributePreferredSharedMemoryCarveout, percent)); +#endif +} + +template +inline auto set_prefer_l1_carveout_with_occupancy( + const int lower_bound_percent, + const int upper_bound_percent, + const uint32_t min_occupancy, + T kernel, + const uint32_t block_threads, + const std::size_t dyn_smem_bytes = 0) -> L1Carveout { +#ifdef USE_ROCM + (void)lower_bound_percent, (void)upper_bound_percent, (void)min_occupancy; + return {-1, get_blocks_per_sm(kernel, block_threads, dyn_smem_bytes)}; +#else + const auto kernel_ptr = reinterpret_cast(kernel); + // The attribute and the occupancy query both act on the current device. + static_assert(cudaSharedmemCarveoutMaxL1 <= cudaSharedmemCarveoutMaxShared); + CHECK_HOST( + min_occupancy > 0 && lower_bound_percent <= upper_bound_percent // + && lower_bound_percent >= cudaSharedmemCarveoutMaxL1 // + && upper_bound_percent <= cudaSharedmemCarveoutMaxShared); + for (int p = lower_bound_percent; p <= upper_bound_percent; ++p) { + set_smem_carveout(kernel_ptr, p); + const auto occupancy = get_blocks_per_sm(kernel_ptr, block_threads, dyn_smem_bytes); + if (occupancy >= min_occupancy) return {p, occupancy}; + } + SGL_PANIC << "no carveout in [" << lower_bound_percent << ", " << upper_bound_percent + << "] can satisfy target occupancy " << min_occupancy << " for kernel " << kernel_ptr; + __builtin_unreachable(); +#endif +} + +/// \brief Prefer the largest L1 carveout that keeps `kernel`'s default occupancy on the current +/// device (PDL secondaries inherit the primary's carveout). The attribute is per device and sticky +/// for the process; panics when no carveout restores occupancy. ROCm has no such attribute and +/// reports percent -1. Wrap it in `init_per_device(device, ...)` to run it once per device. +template +inline auto set_prefer_l1_carveout(T kernel, uint32_t block_threads, std::size_t dyn_smem_bytes = 0) -> L1Carveout { +#ifdef USE_ROCM + return {-1, get_blocks_per_sm(kernel, block_threads, dyn_smem_bytes)}; +#else + const auto kernel_ptr = reinterpret_cast(kernel); + set_smem_carveout(kernel_ptr, cudaSharedmemCarveoutDefault); + const auto default_occupancy = get_blocks_per_sm(kernel_ptr, block_threads, dyn_smem_bytes); + return set_prefer_l1_carveout_with_occupancy( + cudaSharedmemCarveoutMaxL1, + cudaSharedmemCarveoutMaxShared, + default_occupancy, + kernel_ptr, + block_threads, + dyn_smem_bytes); +#endif +} + +/// \brief Opt `kernel` into `dyn_smem_bytes` of dynamic shared memory on the current device. +template +inline void set_max_dynamic_smem(T kernel, std::size_t dyn_smem_bytes) { + const auto kernel_ptr = reinterpret_cast(kernel); + const auto num_bytes = static_cast(dyn_smem_bytes); + CHECK_CUDA(::cudaFuncSetAttribute(kernel_ptr, cudaFuncAttributeMaxDynamicSharedMemorySize, num_bytes)); +} + +/// \brief Run `fn` on the first call per device, keyed off the uniqueness of (`Tag`, `Fn`). +/// \note Relaxed on purpose, to keep the hot path a single atomic load: concurrent first callers +/// may all run `fn`, so it must be idempotent (e.g. setting a function attribute). A throwing `fn` +/// leaves the device uninitialized, and devices past `kNumStaticMaxDevice` rerun `fn` every call. +template +inline void init_per_device(int device_id, bool use_cache, Fn&& fn) { + static_assert(kNumStaticMaxDevice % 8 == 0); + constexpr uint32_t kNumBytes = kNumStaticMaxDevice / 8; + static std::atomic_uint8_t s_initialized[kNumBytes]{}; + const auto device = static_cast(device_id); + const auto can_use_cache = use_cache && device < kNumStaticMaxDevice; + const auto mask = static_cast(1u << (device % 8)); + if (can_use_cache && s_initialized[device / 8].load() & mask) return; + std::forward(fn)(); + if (can_use_cache) s_initialized[device / 8].fetch_or(mask); +} + +// Some wrappers of the above functions + +template +inline void init_per_device(details::MaybeDevice device, Fn&& fn) { + return init_per_device(device.device_id, true, std::forward(fn)); +} + +template +inline void set_max_dynamic_smem_per_device(details::MaybeDevice device, std::size_t dyn_smem_bytes) { + using Tag = details::Tag; + return init_per_device(device.device_id, true, [=] { set_max_dynamic_smem(kernel, dyn_smem_bytes); }); +} + } // namespace host::runtime } // namespace sglang diff --git a/python/sglang/kernels/jit/include/sgl_kernel/utils.cuh b/python/sglang/kernels/jit/include/sgl_kernel/utils.cuh index fa724a9938..d69de6c302 100644 --- a/python/sglang/kernels/jit/include/sgl_kernel/utils.cuh +++ b/python/sglang/kernels/jit/include/sgl_kernel/utils.cuh @@ -11,7 +11,6 @@ /// - Typed `load_as` / `store_as` for void-pointer access. /// - `pointer::offset` for safe void-pointer arithmetic. /// - `host::LaunchKernel` - kernel launcher with optional PDL. -/// - `host::prefer_l1_carveout` / `host::ensure_prefer_l1` - occupancy-preserving L1 carveout preference. /// - `host::RuntimeDeviceCheck` - CUDA error checking. #pragma once @@ -25,14 +24,10 @@ #include #include #include -#include -#include #include #include -#include #ifndef USE_ROCM -#include - +#include #include #include #include @@ -59,10 +54,6 @@ inline constexpr auto cudaSuccess = hipSuccess; #define cudaDeviceGetAttribute hipDeviceGetAttribute #define cudaDevAttrComputeCapabilityMajor hipDeviceAttributeComputeCapabilityMajor #define cudaDevAttrComputeCapabilityMinor hipDeviceAttributeComputeCapabilityMinor -#define cudaGetDevice hipGetDevice -#ifndef cudaOccupancyMaxActiveBlocksPerMultiprocessor -#define cudaOccupancyMaxActiveBlocksPerMultiprocessor hipOccupancyMaxActiveBlocksPerMultiprocessor -#endif #define cudaFuncSetAttribute hipFuncSetAttribute #define cudaFuncAttributeMaxDynamicSharedMemorySize hipFuncAttributeMaxDynamicSharedMemorySize #endif @@ -337,13 +328,42 @@ SGL_DEVICE_HOST constexpr uint32_t get_tmem_cols(uint32_t num_cols) { namespace host { +namespace runtime { + +inline bool is_cuda_error(::cudaError_t error) { + return error != ::cudaSuccess; +} + +inline const char* get_cuda_error_string(::cudaError_t error) { + return ::cudaGetErrorString(error); +} + +#ifndef USE_ROCM + +inline bool is_cuda_error(::CUresult error) { + return error != ::CUDA_SUCCESS; +} + +inline const char* get_cuda_error_string(::CUresult error) { + const char* error_string = ""; + // cuGetErrorString only fails with CUDA_ERROR_INVALID_VALUE, leaving the string null. + const auto new_error = ::cuGetErrorString(error, &error_string); + return new_error == ::CUDA_SUCCESS ? error_string : "Unknown error"; +} + +#endif + +} // namespace runtime + /** - * \brief Check the CUDA error code and panic with location info on failure. + * \brief Check a CUDA runtime (`cudaError_t`) or driver (`CUresult`) error code + * and panic with location info on failure. */ -inline void RuntimeDeviceCheck(::cudaError_t error, DebugInfo location = {}) { - if (error != ::cudaSuccess) { +template +inline void RuntimeDeviceCheck(T error, DebugInfo location = {}) { + if (runtime::is_cuda_error(error)) { [[unlikely]]; - host::panic(location, "CUDA error: ", ::cudaGetErrorString(error)); + panic(location, "CUDA error: ", runtime::get_cuda_error_string(error)); } } @@ -352,58 +372,6 @@ inline void RuntimeDeviceCheck(DebugInfo location = {}) { return RuntimeDeviceCheck(::cudaGetLastError(), location); } -/// \brief Where `prefer_l1_carveout` settled: the carveout percentage and the kernel's blocks/SM there. -struct L1Carveout { - int carveout_pct; - uint32_t blocks_per_sm; -}; - -/// \brief Prefer the largest L1 carveout that keeps `kernel`'s default occupancy on `device_id` -/// (PDL secondaries inherit the primary's carveout). Per device, sticky for the process; panics -/// when no carveout restores occupancy. ROCm has no such attribute and reports pct -1. -template -inline auto prefer_l1_carveout(T&& kernel, int device_id, uint32_t block_threads, std::size_t dyn_smem_bytes = 0) - -> L1Carveout { - const auto blocks_per_sm = [&] { - int blocks = 0; - RuntimeDeviceCheck(::cudaOccupancyMaxActiveBlocksPerMultiprocessor(&blocks, kernel, block_threads, dyn_smem_bytes)); - return static_cast(blocks); - }; -#ifdef USE_ROCM - (void)device_id; - return {-1, blocks_per_sm()}; -#else - // The attribute and the occupancy query both act on the current device. - tvm::ffi::CUDADeviceGuard guard(device_id); - const auto set_carveout = [&](int pct) { - RuntimeDeviceCheck(::cudaFuncSetAttribute(kernel, cudaFuncAttributePreferredSharedMemoryCarveout, pct)); - }; - set_carveout(cudaSharedmemCarveoutDefault); - const uint32_t occupancy = blocks_per_sm(); - RuntimeCheck(occupancy > 0, "kernel does not fit on an SM"); - for (int pct = cudaSharedmemCarveoutMaxL1;; ++pct) { - set_carveout(pct); - if (const uint32_t now = blocks_per_sm(); now >= occupancy) return {pct, now}; - RuntimeCheck(pct < cudaSharedmemCarveoutMaxShared, "no carveout restores occupancy ", occupancy); - } -#endif -} - -/// \brief `prefer_l1_carveout` memoized per (kernel, device): the attribute is sticky, so a hit returns the -/// settled value without touching the device. Load-time configure() and the launch builder share the memo. -/// Callers must keep the block size and dynamic shared memory fixed for each (kernel, device), and -/// must not change the kernel's carveout attribute after configuration. -template -inline auto ensure_prefer_l1(T kernel, int device_id, uint32_t block_threads, std::size_t dyn_smem_bytes = 0) - -> L1Carveout { - static std::mutex mutex; - static std::map, L1Carveout> settled; - const std::lock_guard lock(mutex); - const auto key = std::make_pair(reinterpret_cast(kernel), device_id); - if (const auto it = settled.find(key); it != settled.end()) return it->second; - return settled.emplace(key, prefer_l1_carveout(kernel, device_id, block_threads, dyn_smem_bytes)).first->second; -} - /** * \brief Kernel launcher with automatic stream resolution and PDL support. * @@ -413,8 +381,6 @@ inline auto ensure_prefer_l1(T kernel, int device_id, uint32_t block_threads, st * .enable_pdl(true)(my_kernel, arg0, arg1); * host::LaunchKernel(grid, block, stream) * .config({.use_pdl = true, .cluster_dim = cluster_dim})(my_kernel, arg0); - * host::LaunchKernel(grid, block, device) - * .config({.use_pdl = true, .prefer_l1 = true})(my_kernel, arg0); * \endcode * * The constructor resolves the CUDA stream from a `DLDevice` (via `TVMFFIEnvGetStream`) @@ -425,7 +391,6 @@ struct LaunchKernel { struct KernelConfig { bool use_pdl = false; std::optional cluster_dim = std::nullopt; - bool prefer_l1 = false; // `ensure_prefer_l1` on the launch device before the first launch }; public: @@ -436,8 +401,7 @@ struct LaunchKernel { std::size_t dynamic_shared_mem_bytes = 0, DebugInfo location = {}) noexcept : m_config(s_make_config(grid_dim, block_dim, resolve_device(device), dynamic_shared_mem_bytes)), - m_location(location), - m_device_id(device.device_id) {} + m_location(location) {} explicit LaunchKernel( dim3 grid_dim, @@ -445,9 +409,7 @@ struct LaunchKernel { cudaStream_t stream, std::size_t dynamic_shared_mem_bytes = 0, DebugInfo location = {}) noexcept - : m_config(s_make_config(grid_dim, block_dim, stream, dynamic_shared_mem_bytes)), - m_location(location), - m_device_id(-1) {} + : m_config(s_make_config(grid_dim, block_dim, stream, dynamic_shared_mem_bytes)), m_location(location) {} LaunchKernel(const LaunchKernel&) = delete; LaunchKernel& operator=(const LaunchKernel&) = delete; @@ -483,11 +445,6 @@ struct LaunchKernel { return *this; } - auto prefer_l1(bool enabled = true) -> LaunchKernel& { - m_prefer_l1 = enabled; - return *this; - } - /** * \brief Configure the kernel launch with the given options. * \param config The kernel configuration options. @@ -499,13 +456,11 @@ struct LaunchKernel { auto config(const KernelConfig& config) -> LaunchKernel& { if (config.use_pdl) this->enable_pdl(true); if (config.cluster_dim) this->enable_cluster(*config.cluster_dim); - if (config.prefer_l1) this->prefer_l1(true); return *this; } template auto operator()(T&& kernel, Args&&... args) const -> void { - if (m_prefer_l1) apply_prefer_l1(kernel); #ifdef USE_ROCM hipLaunchKernelGGL( std::forward(kernel), @@ -540,28 +495,15 @@ struct LaunchKernel { return config; } - // Memo hit after load-time configure(); stream-constructed launches use the current device. - template - void apply_prefer_l1(T&& kernel) const { - int device_id = m_device_id; - if (device_id < 0) RuntimeDeviceCheck(::cudaGetDevice(&device_id)); - const dim3 block = m_config.blockDim; - ensure_prefer_l1(+kernel, device_id, block.x * block.y * block.z, m_config.dynamicSmemBytes); - } - cudaLaunchConfig_t m_config; const DebugInfo m_location; cudaLaunchAttribute m_attrs[2]; - int m_device_id; - bool m_prefer_l1 = false; }; -// The empty-true-branch if/else form keeps a trailing `else` in user code -// bound to the user's `if`, not to the macro's. -#define CHECK_CUDA(COND) \ - if (const auto error = (COND); error == ::cudaSuccess) [[likely]] { \ - } else \ - host::Error() << "CUDA error: " << ::cudaGetErrorString(error) << ". " +#define CHECK_CUDA(COND) \ + if (const auto _err = (COND); !::sglang::host::runtime::is_cuda_error(_err)) [[likely]] { \ + } else \ + SGL_PANIC << "CUDA error: " << ::sglang::host::runtime::get_cuda_error_string(_err) << ". " } // namespace host diff --git a/python/sglang/kernels/jit/include/sgl_kernel/utils.h b/python/sglang/kernels/jit/include/sgl_kernel/utils.h index 9c7f29ddce..38ad89ba13 100644 --- a/python/sglang/kernels/jit/include/sgl_kernel/utils.h +++ b/python/sglang/kernels/jit/include/sgl_kernel/utils.h @@ -178,37 +178,28 @@ inline auto irange(T start, T end) { /** \brief Error class for stream-style error logging. */ struct Error { - Error(DebugInfo location = {}) { - m_oss << "Failed at " << location.file_name() << ":" << location.line() << ": "; + std::ostringstream stream; + explicit Error(DebugInfo location = {}) { + stream << "Failed at " << location.file_name() << ":" << location.line() << ": "; } - - template - Error& operator<<(T&& arg) { - m_oss << std::forward(arg); - return *this; - } - [[noreturn]] ~Error() noexcept(false) { - throw PanicError(std::move(m_oss).str()); + throw PanicError(std::move(stream).str()); } - - private: - std::ostringstream m_oss; }; +#define SGL_PANIC ::sglang::host::Error().stream + /** * \brief 0-overhead CHECK macro for host code. This can avoid unnecessary * instantiation of error messages when the condition is true. * * Usage: CHECK_HOST(ptr != nullptr) << "Pointer must not be null"; */ -// The empty-true-branch if/else form keeps a trailing `else` in user code -// bound to the user's `if`, not to the macro's. #define CHECK_HOST(COND) \ if (COND) [[likely]] { \ } else \ - host::Error() + SGL_PANIC } // namespace host diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index 77b16ef36f..c1bdf79f56 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -353,6 +353,8 @@ class Envs: SGLANG_LOG_REQUEST_HEADERS = EnvTuple(tuple()) SGLANG_LOG_SCHEDULER_STATUS_TARGET = EnvStr("") SGLANG_LOG_SCHEDULER_STATUS_INTERVAL = EnvFloat(60.0) + # Read by JIT kernel headers (sgl_kernel/logging.h): DEBUG | INFO | WARNING | ERROR. + SGLANG_JIT_LOG_LEVEL = EnvStr("WARNING") SGLANG_ENABLE_RANK_CONSENSUS_CHECKER = EnvBool(False) # =================================================================== diff --git a/test/registered/kernels/ops/elementwise/launch_carveout.cuh b/test/registered/kernels/ops/elementwise/launch_carveout.cuh deleted file mode 100644 index adb9ab1f6c..0000000000 --- a/test/registered/kernels/ops/elementwise/launch_carveout.cuh +++ /dev/null @@ -1,64 +0,0 @@ -#include -#include - -namespace sglang { - -template -__global__ void carveout_probe(int32_t *output) { - device::PDLWaitPrimary(); - extern __shared__ int32_t scratch[]; - if constexpr (kSmem > 0) { - scratch[threadIdx.x] = threadIdx.x; - __syncthreads(); - output[threadIdx.x] = scratch[255 - threadIdx.x]; - } else { - output[threadIdx.x] = 255 - threadIdx.x; - } -} - -template -void check_launch_carveout(tvm::ffi::TensorView output) { - host::TensorMatcher({256}) - .with_dtype() - .with_device() - .verify(output); - const auto kernel = carveout_probe; - const int device_id = output.device().device_id; - const tvm::ffi::CUDADeviceGuard guard(device_id); - int baseline = 0; - CHECK_CUDA(cudaOccupancyMaxActiveBlocksPerMultiprocessor(&baseline, kernel, - 256, kSmem)); - CHECK_HOST(baseline > 0); - if constexpr (kMode == 2) - host::ensure_prefer_l1(kernel, device_id, 256, kSmem); - auto *ptr = static_cast(output.data_ptr()); - if constexpr (kMode == 0) { - host::LaunchKernel(1, 256, output.device(), kSmem)(kernel, ptr); - } else if constexpr (kMode == 1) { - host::LaunchKernel( - 1, 256, host::LaunchKernel::resolve_device(output.device()), kSmem) - .prefer_l1()(kernel, ptr); - } else { - host::LaunchKernel(1, 256, output.device(), kSmem) - .config({.use_pdl = true, .prefer_l1 = true})(kernel, ptr); - } - cudaFuncAttributes attrs{}; - CHECK_CUDA(cudaFuncGetAttributes(&attrs, kernel)); - if constexpr (kMode == 0) { - CHECK_HOST(attrs.preferredShmemCarveout == cudaSharedmemCarveoutDefault); - } else { - const auto receipt = host::ensure_prefer_l1(kernel, device_id, 256, kSmem); - const auto memo = host::ensure_prefer_l1(kernel, device_id, 256, kSmem); - CHECK_HOST(receipt.carveout_pct >= 0 && receipt.carveout_pct <= 100); - CHECK_HOST(receipt.blocks_per_sm >= static_cast(baseline)); - CHECK_HOST(attrs.preferredShmemCarveout == receipt.carveout_pct); - CHECK_HOST(memo.carveout_pct == receipt.carveout_pct && - memo.blocks_per_sm == receipt.blocks_per_sm); - int actual = 0; - CHECK_CUDA(cudaOccupancyMaxActiveBlocksPerMultiprocessor(&actual, kernel, - 256, kSmem)); - CHECK_HOST(actual >= baseline); - } -} - -} // namespace sglang diff --git a/test/registered/kernels/ops/elementwise/test_launch_carveout.py b/test/registered/kernels/ops/elementwise/test_launch_carveout.py deleted file mode 100644 index d49461efb0..0000000000 --- a/test/registered/kernels/ops/elementwise/test_launch_carveout.py +++ /dev/null @@ -1,45 +0,0 @@ -"""L1 preference must retain default occupancy and leave other devices untouched.""" - -import pathlib -import sys - -import pytest -import torch - -from sglang.kernels.jit.utils import load_jit -from sglang.test.ci.ci_register import register_cuda_ci - -register_cuda_ci(est_time=10, stage="base-b-kernel-unit", runner_config="1-gpu-large") - - -def test_launch_carveout(): - """A MaxL1-only policy loses occupancy for shared-memory-heavy kernels.""" - current = torch.cuda.current_device() - untouched = [ - device - for device in range(torch.cuda.device_count()) - if device != current and not torch._C._cuda_hasPrimaryContext(device) - ] - cases = [(smem, mode) for smem in (0, 32768) for mode in range(3)] - module = load_jit( - "test_launch_carveout", - cuda_files=[str(pathlib.Path(__file__).with_name("launch_carveout.cuh"))], - cuda_wrappers=[ - (f"run_{smem}_{mode}", f"check_launch_carveout<{smem}, {mode}>") - for smem, mode in cases - ], - ) - expected = torch.arange(255, -1, -1, dtype=torch.int32, device="cuda") - output = torch.empty_like(expected) - for smem, mode in cases: - run = module[f"run_{smem}_{mode}"] - for _ in range(2): - output.fill_(-1) - run(output) - assert torch.equal(output, expected) - for device in untouched: - assert not torch._C._cuda_hasPrimaryContext(device) - - -if __name__ == "__main__": - sys.exit(pytest.main([__file__]))