From 2909f84e3b2fc627bbc9de76ac71d1c295639a8c Mon Sep 17 00:00:00 2001 From: metamergebot Date: Wed, 23 Sep 2026 11:49:34 -0700 Subject: [PATCH] [JIT] Add an occupancy-preserving L1 carveout preference (#40767) Co-authored-by: metamergebot <324680979+metamergebot@users.noreply.github.com> Co-authored-by: Oguz Ulgen --- .../kernels/jit/include/sgl_kernel/utils.cuh | 90 ++++++++++++++++++- .../ops/elementwise/launch_carveout.cuh | 64 +++++++++++++ .../ops/elementwise/test_launch_carveout.py | 45 ++++++++++ 3 files changed, 197 insertions(+), 2 deletions(-) create mode 100644 test/registered/kernels/ops/elementwise/launch_carveout.cuh create mode 100644 test/registered/kernels/ops/elementwise/test_launch_carveout.py diff --git a/python/sglang/kernels/jit/include/sgl_kernel/utils.cuh b/python/sglang/kernels/jit/include/sgl_kernel/utils.cuh index c127f76b75..fa724a9938 100644 --- a/python/sglang/kernels/jit/include/sgl_kernel/utils.cuh +++ b/python/sglang/kernels/jit/include/sgl_kernel/utils.cuh @@ -11,6 +11,7 @@ /// - Typed `load_as` / `store_as` for void-pointer access. /// - `pointer::offset` for safe void-pointer arithmetic. /// - `host::LaunchKernel` - kernel launcher with optional PDL. +/// - `host::prefer_l1_carveout` / `host::ensure_prefer_l1` - occupancy-preserving L1 carveout preference. /// - `host::RuntimeDeviceCheck` - CUDA error checking. #pragma once @@ -24,9 +25,14 @@ #include #include #include +#include +#include #include #include +#include #ifndef USE_ROCM +#include + #include #include #include @@ -53,6 +59,10 @@ inline constexpr auto cudaSuccess = hipSuccess; #define cudaDeviceGetAttribute hipDeviceGetAttribute #define cudaDevAttrComputeCapabilityMajor hipDeviceAttributeComputeCapabilityMajor #define cudaDevAttrComputeCapabilityMinor hipDeviceAttributeComputeCapabilityMinor +#define cudaGetDevice hipGetDevice +#ifndef cudaOccupancyMaxActiveBlocksPerMultiprocessor +#define cudaOccupancyMaxActiveBlocksPerMultiprocessor hipOccupancyMaxActiveBlocksPerMultiprocessor +#endif #define cudaFuncSetAttribute hipFuncSetAttribute #define cudaFuncAttributeMaxDynamicSharedMemorySize hipFuncAttributeMaxDynamicSharedMemorySize #endif @@ -342,6 +352,58 @@ inline void RuntimeDeviceCheck(DebugInfo location = {}) { return RuntimeDeviceCheck(::cudaGetLastError(), location); } +/// \brief Where `prefer_l1_carveout` settled: the carveout percentage and the kernel's blocks/SM there. +struct L1Carveout { + int carveout_pct; + uint32_t blocks_per_sm; +}; + +/// \brief Prefer the largest L1 carveout that keeps `kernel`'s default occupancy on `device_id` +/// (PDL secondaries inherit the primary's carveout). Per device, sticky for the process; panics +/// when no carveout restores occupancy. ROCm has no such attribute and reports pct -1. +template +inline auto prefer_l1_carveout(T&& kernel, int device_id, uint32_t block_threads, std::size_t dyn_smem_bytes = 0) + -> L1Carveout { + const auto blocks_per_sm = [&] { + int blocks = 0; + RuntimeDeviceCheck(::cudaOccupancyMaxActiveBlocksPerMultiprocessor(&blocks, kernel, block_threads, dyn_smem_bytes)); + return static_cast(blocks); + }; +#ifdef USE_ROCM + (void)device_id; + return {-1, blocks_per_sm()}; +#else + // The attribute and the occupancy query both act on the current device. + tvm::ffi::CUDADeviceGuard guard(device_id); + const auto set_carveout = [&](int pct) { + RuntimeDeviceCheck(::cudaFuncSetAttribute(kernel, cudaFuncAttributePreferredSharedMemoryCarveout, pct)); + }; + set_carveout(cudaSharedmemCarveoutDefault); + const uint32_t occupancy = blocks_per_sm(); + RuntimeCheck(occupancy > 0, "kernel does not fit on an SM"); + for (int pct = cudaSharedmemCarveoutMaxL1;; ++pct) { + set_carveout(pct); + if (const uint32_t now = blocks_per_sm(); now >= occupancy) return {pct, now}; + RuntimeCheck(pct < cudaSharedmemCarveoutMaxShared, "no carveout restores occupancy ", occupancy); + } +#endif +} + +/// \brief `prefer_l1_carveout` memoized per (kernel, device): the attribute is sticky, so a hit returns the +/// settled value without touching the device. Load-time configure() and the launch builder share the memo. +/// Callers must keep the block size and dynamic shared memory fixed for each (kernel, device), and +/// must not change the kernel's carveout attribute after configuration. +template +inline auto ensure_prefer_l1(T kernel, int device_id, uint32_t block_threads, std::size_t dyn_smem_bytes = 0) + -> L1Carveout { + static std::mutex mutex; + static std::map, L1Carveout> settled; + const std::lock_guard lock(mutex); + const auto key = std::make_pair(reinterpret_cast(kernel), device_id); + if (const auto it = settled.find(key); it != settled.end()) return it->second; + return settled.emplace(key, prefer_l1_carveout(kernel, device_id, block_threads, dyn_smem_bytes)).first->second; +} + /** * \brief Kernel launcher with automatic stream resolution and PDL support. * @@ -351,6 +413,8 @@ inline void RuntimeDeviceCheck(DebugInfo location = {}) { * .enable_pdl(true)(my_kernel, arg0, arg1); * host::LaunchKernel(grid, block, stream) * .config({.use_pdl = true, .cluster_dim = cluster_dim})(my_kernel, arg0); + * host::LaunchKernel(grid, block, device) + * .config({.use_pdl = true, .prefer_l1 = true})(my_kernel, arg0); * \endcode * * The constructor resolves the CUDA stream from a `DLDevice` (via `TVMFFIEnvGetStream`) @@ -361,6 +425,7 @@ struct LaunchKernel { struct KernelConfig { bool use_pdl = false; std::optional cluster_dim = std::nullopt; + bool prefer_l1 = false; // `ensure_prefer_l1` on the launch device before the first launch }; public: @@ -371,7 +436,8 @@ struct LaunchKernel { std::size_t dynamic_shared_mem_bytes = 0, DebugInfo location = {}) noexcept : m_config(s_make_config(grid_dim, block_dim, resolve_device(device), dynamic_shared_mem_bytes)), - m_location(location) {} + m_location(location), + m_device_id(device.device_id) {} explicit LaunchKernel( dim3 grid_dim, @@ -379,7 +445,9 @@ struct LaunchKernel { cudaStream_t stream, std::size_t dynamic_shared_mem_bytes = 0, DebugInfo location = {}) noexcept - : m_config(s_make_config(grid_dim, block_dim, stream, dynamic_shared_mem_bytes)), m_location(location) {} + : m_config(s_make_config(grid_dim, block_dim, stream, dynamic_shared_mem_bytes)), + m_location(location), + m_device_id(-1) {} LaunchKernel(const LaunchKernel&) = delete; LaunchKernel& operator=(const LaunchKernel&) = delete; @@ -415,6 +483,11 @@ struct LaunchKernel { return *this; } + auto prefer_l1(bool enabled = true) -> LaunchKernel& { + m_prefer_l1 = enabled; + return *this; + } + /** * \brief Configure the kernel launch with the given options. * \param config The kernel configuration options. @@ -426,11 +499,13 @@ struct LaunchKernel { auto config(const KernelConfig& config) -> LaunchKernel& { if (config.use_pdl) this->enable_pdl(true); if (config.cluster_dim) this->enable_cluster(*config.cluster_dim); + if (config.prefer_l1) this->prefer_l1(true); return *this; } template auto operator()(T&& kernel, Args&&... args) const -> void { + if (m_prefer_l1) apply_prefer_l1(kernel); #ifdef USE_ROCM hipLaunchKernelGGL( std::forward(kernel), @@ -465,9 +540,20 @@ struct LaunchKernel { return config; } + // Memo hit after load-time configure(); stream-constructed launches use the current device. + template + void apply_prefer_l1(T&& kernel) const { + int device_id = m_device_id; + if (device_id < 0) RuntimeDeviceCheck(::cudaGetDevice(&device_id)); + const dim3 block = m_config.blockDim; + ensure_prefer_l1(+kernel, device_id, block.x * block.y * block.z, m_config.dynamicSmemBytes); + } + cudaLaunchConfig_t m_config; const DebugInfo m_location; cudaLaunchAttribute m_attrs[2]; + int m_device_id; + bool m_prefer_l1 = false; }; // The empty-true-branch if/else form keeps a trailing `else` in user code diff --git a/test/registered/kernels/ops/elementwise/launch_carveout.cuh b/test/registered/kernels/ops/elementwise/launch_carveout.cuh new file mode 100644 index 0000000000..adb9ab1f6c --- /dev/null +++ b/test/registered/kernels/ops/elementwise/launch_carveout.cuh @@ -0,0 +1,64 @@ +#include +#include + +namespace sglang { + +template +__global__ void carveout_probe(int32_t *output) { + device::PDLWaitPrimary(); + extern __shared__ int32_t scratch[]; + if constexpr (kSmem > 0) { + scratch[threadIdx.x] = threadIdx.x; + __syncthreads(); + output[threadIdx.x] = scratch[255 - threadIdx.x]; + } else { + output[threadIdx.x] = 255 - threadIdx.x; + } +} + +template +void check_launch_carveout(tvm::ffi::TensorView output) { + host::TensorMatcher({256}) + .with_dtype() + .with_device() + .verify(output); + const auto kernel = carveout_probe; + const int device_id = output.device().device_id; + const tvm::ffi::CUDADeviceGuard guard(device_id); + int baseline = 0; + CHECK_CUDA(cudaOccupancyMaxActiveBlocksPerMultiprocessor(&baseline, kernel, + 256, kSmem)); + CHECK_HOST(baseline > 0); + if constexpr (kMode == 2) + host::ensure_prefer_l1(kernel, device_id, 256, kSmem); + auto *ptr = static_cast(output.data_ptr()); + if constexpr (kMode == 0) { + host::LaunchKernel(1, 256, output.device(), kSmem)(kernel, ptr); + } else if constexpr (kMode == 1) { + host::LaunchKernel( + 1, 256, host::LaunchKernel::resolve_device(output.device()), kSmem) + .prefer_l1()(kernel, ptr); + } else { + host::LaunchKernel(1, 256, output.device(), kSmem) + .config({.use_pdl = true, .prefer_l1 = true})(kernel, ptr); + } + cudaFuncAttributes attrs{}; + CHECK_CUDA(cudaFuncGetAttributes(&attrs, kernel)); + if constexpr (kMode == 0) { + CHECK_HOST(attrs.preferredShmemCarveout == cudaSharedmemCarveoutDefault); + } else { + const auto receipt = host::ensure_prefer_l1(kernel, device_id, 256, kSmem); + const auto memo = host::ensure_prefer_l1(kernel, device_id, 256, kSmem); + CHECK_HOST(receipt.carveout_pct >= 0 && receipt.carveout_pct <= 100); + CHECK_HOST(receipt.blocks_per_sm >= static_cast(baseline)); + CHECK_HOST(attrs.preferredShmemCarveout == receipt.carveout_pct); + CHECK_HOST(memo.carveout_pct == receipt.carveout_pct && + memo.blocks_per_sm == receipt.blocks_per_sm); + int actual = 0; + CHECK_CUDA(cudaOccupancyMaxActiveBlocksPerMultiprocessor(&actual, kernel, + 256, kSmem)); + CHECK_HOST(actual >= baseline); + } +} + +} // namespace sglang diff --git a/test/registered/kernels/ops/elementwise/test_launch_carveout.py b/test/registered/kernels/ops/elementwise/test_launch_carveout.py new file mode 100644 index 0000000000..d49461efb0 --- /dev/null +++ b/test/registered/kernels/ops/elementwise/test_launch_carveout.py @@ -0,0 +1,45 @@ +"""L1 preference must retain default occupancy and leave other devices untouched.""" + +import pathlib +import sys + +import pytest +import torch + +from sglang.kernels.jit.utils import load_jit +from sglang.test.ci.ci_register import register_cuda_ci + +register_cuda_ci(est_time=10, stage="base-b-kernel-unit", runner_config="1-gpu-large") + + +def test_launch_carveout(): + """A MaxL1-only policy loses occupancy for shared-memory-heavy kernels.""" + current = torch.cuda.current_device() + untouched = [ + device + for device in range(torch.cuda.device_count()) + if device != current and not torch._C._cuda_hasPrimaryContext(device) + ] + cases = [(smem, mode) for smem in (0, 32768) for mode in range(3)] + module = load_jit( + "test_launch_carveout", + cuda_files=[str(pathlib.Path(__file__).with_name("launch_carveout.cuh"))], + cuda_wrappers=[ + (f"run_{smem}_{mode}", f"check_launch_carveout<{smem}, {mode}>") + for smem, mode in cases + ], + ) + expected = torch.arange(255, -1, -1, dtype=torch.int32, device="cuda") + output = torch.empty_like(expected) + for smem, mode in cases: + run = module[f"run_{smem}_{mode}"] + for _ in range(2): + output.fill_(-1) + run(output) + assert torch.equal(output, expected) + for device in untouched: + assert not torch._C._cuda_hasPrimaryContext(device) + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__]))