[DeepSeek-V4.1] DeepSelect cluster top-k for long rows and bf16 index logits (#44051)

Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Co-authored-by: Xiaoyu Zhang <1182563586@qq.com>
This commit is contained in:
Ziyi Xu
2026-10-15 09:00:19 +08:00
committed by GitHub
co-authored by Claude Opus 5.5 Xiaoyu Zhang
parent c5c4bbbd25
commit 9d68ecce2e
16 changed files with 1265 additions and 348 deletions
@@ -649,9 +649,10 @@ public:
// Save result to `smem.reconstruct_pivot_bucket` and `smem.reconstruct_num_should_select`
// If `CHECK_IF_SHOULD_SELECT_WHOLE_BUCKET`, return whether we should select all elements in that bucket.
// Should be called by a whole warp
template<bool CHECK_IF_SHOULD_SELECT_WHOLE_BUCKET>
// [SGLang] Any plan with `reconstruct_pivot_bucket` and `reconstruct_num_should_select` will do
template<bool CHECK_IF_SHOULD_SELECT_WHOLE_BUCKET, typename PlanT>
static __device__ __forceinline__
bool find_pivot_in_histogram(SharedMemoryPlanBase &smem, const uint32_t *bucket_counter, uint32_t topk, uint32_t lane_idx) {
bool find_pivot_in_histogram(PlanT &smem, const uint32_t *bucket_counter, uint32_t topk, uint32_t lane_idx) {
bool should_select_whole_bucket = false;
// Each lane loads 8 counters
static_assert(NUM_RECONSTRUCT_BUCKETS == 32 * 8);
@@ -954,8 +955,10 @@ public:
);
}
// Take the appropriate action when we see a NaN (i.e. `trap()` or write 0x3f3f3f3f)
static __device__ __forceinline__
// Take the appropriate action when we see a NaN: `trap()`, or [SGLang] write the row as if every selected
// index were 0, so downstream gathers stay in range. Called by the whole CTA; out of line, since inlined copies
// of this cold path only cost instruction fetch on every launch.
static __device__ __noinline__
void take_action_when_have_nan(const TopkSelectArgs &args, uint32_t batch_idx) {
if (args.abort_when_nan_found) {
if (threadIdx.x == 0) {
@@ -964,8 +967,19 @@ public:
__syncthreads();
ku::trap();
} else {
if (threadIdx.x == 0) {
*((OutIdxT*)args.output_index + batch_idx * args.stride_output_index_batch) = 0x3F3F3F3F;
OutIdxT* result_indices = (OutIdxT*)args.output_index + (uint64_t)batch_idx * args.stride_output_index_batch;
OutIdxT index0;
if constexpr (Config::page_transform) {
const int* page_table_row = args.page_table + (uint64_t)batch_idx * args.stride_page_table_batch;
index0 = (OutIdxT)EpilogueT::page_transform_index(0u, true, 0, page_table_row, args.page_bits);
} else {
index0 = (OutIdxT)(args.output_idx_offset != nullptr ? __ldg(args.output_idx_offset + batch_idx) : 0);
}
for (uint32_t i = threadIdx.x; i < args.topk; i += NUM_THREADS) result_indices[i] = index0;
if constexpr (Config::return_value) {
const ValueT value0 = ((const ValueT*)args.input)[(uint64_t)batch_idx * args.stride_input_batch];
ValueT* result_values = (ValueT*)args.output_value + (uint64_t)batch_idx * args.stride_output_value_batch;
for (uint32_t i = threadIdx.x; i < args.topk; i += NUM_THREADS) result_values[i] = value0;
}
}
}
File diff suppressed because it is too large Load Diff
@@ -9,30 +9,39 @@
#include <dlpack/dlpack.h>
#include <tvm/ffi/container/tensor.h>
#include <algorithm>
#include <cstdint>
#include <limits>
#include <type_traits>
namespace sglang {
/// Level-one keys of the two-level indexer: the max of each kBlockTokens-score
/// block, the row's newest block forced to +inf. Contract: BlockAmaxKernel.
/// block, the row's newest block forced to +inf. Scores and keys are each fp32
/// or bf16; a bf16 score's max is exact in either key type. Contract: BlockAmaxKernel.
template <typename InT, typename OutT>
struct BlockAmaxConfig {
using DType = float;
using DType = InT;
using KeyT = OutT;
static constexpr uint32_t kBlockTokens = 8; // scores per key
static constexpr uint32_t kBlockSize = 512;
static constexpr uint32_t kNumItems = 2; // keys per thread
static constexpr uint32_t kOccupancy = 4;
static constexpr uint32_t kKeysPerCTA = kBlockSize * kNumItems;
// One block is 32 B: a single load on Blackwell, two 16 B loads before it.
static constexpr uint32_t kVecSize = device::kMaxVecBytes / sizeof(DType);
// One fp32 block is 32 B: a single load on Blackwell, two 16 B loads before it.
// A bf16 block is one 16 B load anywhere.
static constexpr uint32_t kBlockBytes = kBlockTokens * sizeof(DType);
static constexpr uint32_t kVecBytes = std::min<uint32_t>(kBlockBytes, device::kMaxVecBytes);
static constexpr uint32_t kVecSize = kVecBytes / sizeof(DType);
static constexpr uint32_t kVecsPerBlock = kBlockTokens / kVecSize;
static constexpr uint32_t kNumItems = 64 / kBlockBytes; // 64B scores (16 registers) per thread
static constexpr uint32_t kKeysPerCTA = kBlockSize * kNumItems;
static_assert(kVecsPerBlock * kVecSize == kBlockTokens);
using vec_t = device::AlignedVector<DType, kVecSize>;
};
template <typename InT, typename OutT>
struct BlockAmaxParams {
const BlockAmaxConfig::DType* __restrict__ scores;
BlockAmaxConfig::DType* __restrict__ amax_scores;
const InT* __restrict__ scores;
OutT* __restrict__ amax_scores;
const int32_t* __restrict__ seq_len;
int64_t stride_scores; // in elements
int64_t stride_amax_scores; // in elements
@@ -41,12 +50,11 @@ struct BlockAmaxParams {
/// grid = (rows, ceil(max_keys / kKeysPerCTA)); a CTA owns kKeysPerCTA consecutive
/// keys of one row, a thread kNumItems keys kBlockSize apart (coalesced loads).
template <bool kUsePDL>
__global__ __launch_bounds__(BlockAmaxConfig::kBlockSize, BlockAmaxConfig::kOccupancy) //
void amax8_varlen_kernel(const __grid_constant__ BlockAmaxParams params) {
template <bool kUsePDL, typename InT, typename OutT>
__global__ __launch_bounds__(BlockAmaxConfig<InT, OutT>::kBlockSize, BlockAmaxConfig<InT, OutT>::kOccupancy) //
void amax8_varlen_kernel(const __grid_constant__ BlockAmaxParams<InT, OutT> params) {
using namespace device;
using C = BlockAmaxConfig;
using T = typename C::DType;
using C = BlockAmaxConfig<InT, OutT>;
using vec_t = typename C::vec_t;
const auto bx = blockIdx.x;
const auto by = blockIdx.y;
@@ -81,25 +89,41 @@ __global__ __launch_bounds__(BlockAmaxConfig::kBlockSize, BlockAmaxConfig::kOccu
for (uint32_t i = 0; i < C::kNumItems; ++i) {
const auto idx = first_key + tx + i * C::kBlockSize;
if (idx < num_keys) {
T key = vec[i][0][0];
// fmaxf and __hmax2 ignore a NaN score, torch.amax would propagate it
if constexpr (std::is_same_v<InT, float>) {
float key = vec[i][0][0];
#pragma unroll
for (uint32_t v = 0; v < C::kVecsPerBlock; ++v) {
for (uint32_t v = 0; v < C::kVecsPerBlock; ++v) {
#pragma unroll
for (uint32_t j = 0; j < C::kVecSize; ++j) {
key = fmaxf(key, vec[i][v][j]); // a NaN score is ignored, torch.amax would propagate it
for (uint32_t j = 0; j < C::kVecSize; ++j) {
key = fmaxf(key, vec[i][v][j]);
}
}
const auto fp32_inf = std::numeric_limits<float>::infinity();
out[idx] = cast<OutT>(idx + 1 == num_keys ? fp32_inf : key);
} else {
const auto vec_ptr = reinterpret_cast<bf16x2_t*>(vec[i][0].data());
static_assert(C::kVecSize % 2 == 0);
bf16x2_t key2 = vec_ptr[0];
#pragma unroll
for (uint32_t v = 1; v < C::kVecsPerBlock * C::kVecSize / 2; ++v) {
key2 = __hmax2(key2, vec_ptr[v]);
}
const auto key = __hmax(key2.x, key2.y);
const auto bf16_inf = cast<bf16_t>(std::numeric_limits<float>::infinity());
out[idx] = cast<OutT>(idx + 1 == num_keys ? bf16_inf : key);
}
out[idx] = idx + 1 == num_keys ? std::numeric_limits<T>::infinity() : key;
}
}
}
/// Host entry: `amax_scores[b, i] = max(scores[b, 8 i : 8 i + 8])` for
/// `i < ceil(seq_len[b] / 8)`, the last of them +inf; rows with at most `topk`
/// blocks untouched. `scores` rows must stay 32 B aligned (stride % 8 == 0).
/// blocks untouched. `scores` (`InT`) rows must keep every block of 8 aligned
/// (stride % 8 == 0); `amax_scores` is `OutT`. Each type is fp32 or bf16.
/// The grid covers `amax_scores`' width, so the caller sizes it for the longest
/// row: `seq_len[b] <= 8 * amax_scores.shape[1]` for every row (not checked).
template <bool kPDL>
template <bool kPDL, typename InT, typename OutT>
struct BlockAmaxKernel {
static void amax8_varlen(
const tvm::ffi::TensorView scores,
@@ -107,40 +131,34 @@ struct BlockAmaxKernel {
const tvm::ffi::TensorView amax_scores,
const uint32_t topk) {
using namespace host;
using C = BlockAmaxConfig;
using C = BlockAmaxConfig<InT, OutT>;
auto B = SymbolicSize{"batch_size"};
auto L = SymbolicSize{"max_seq_len"};
auto S = SymbolicSize{"stride_scores"};
auto K = SymbolicSize{"max_keys"};
auto O = SymbolicSize{"stride_amax_scores"};
auto device_ = SymbolicDevice{};
device_.set_options<kDLGPU>();
TensorMatcher({B, L}) // scores
.with_strides({S, 1})
.with_strides({-1, 1})
.with_dtype<typename C::DType>()
.with_device(device_)
.ensure_alignment(C::kBlockBytes)
.verify(scores);
TensorMatcher({B}) // seq_lens
.with_dtype<int32_t>()
.with_device(device_)
.verify(seq_lens);
TensorMatcher({B, K}) // amax_scores
.with_strides({O, 1})
.with_dtype<typename C::DType>()
.with_strides({-1, 1})
.with_dtype<typename C::KeyT>()
.with_device(device_)
.verify(amax_scores);
RuntimeCheck(S.unwrap() % C::kBlockTokens == 0, "stride_scores must keep every block 32 B aligned");
RuntimeCheck(
reinterpret_cast<uintptr_t>(scores.data_ptr()) % (C::kBlockTokens * sizeof(typename C::DType)) == 0,
"scores must be 32 B aligned");
RuntimeCheck(K.unwrap() > 0, "amax_scores must hold at least one key per row");
const auto max_keys = K.unwrap(); // ceil(longest row / 8), sized by the caller
const auto params = BlockAmaxParams{
.scores = static_cast<const typename C::DType*>(scores.data_ptr()),
.amax_scores = static_cast<typename C::DType*>(amax_scores.data_ptr()),
const auto params = BlockAmaxParams<InT, OutT>{
.scores = static_cast<const InT*>(scores.data_ptr()),
.amax_scores = static_cast<OutT*>(amax_scores.data_ptr()),
.seq_len = static_cast<const int32_t*>(seq_lens.data_ptr()),
.stride_scores = S.unwrap(),
.stride_amax_scores = O.unwrap(),
.stride_scores = scores.stride(0),
.stride_amax_scores = amax_scores.stride(0),
.topk = topk,
};
const auto grid = dim3(
@@ -148,7 +166,7 @@ struct BlockAmaxKernel {
static_cast<uint32_t>(div_ceil(max_keys, static_cast<int64_t>(C::kKeysPerCTA))));
LaunchKernel(grid, C::kBlockSize, device_.unwrap())
.config({.use_pdl = kPDL})
.launch(amax8_varlen_kernel<kPDL>, params);
.launch(amax8_varlen_kernel<kPDL, InT, OutT>, params);
}
};
@@ -30,24 +30,26 @@ uint32_t get_max_active_clusters(uint32_t cluster_size, uint32_t num_waves) {
const auto smem = (std::min(budget - std::min(budget, reserved), static_cast<uint32_t>(smem_per_block))) & ~1023u;
const auto num_warps = std::max(1u, std::min(1024u, static_cast<uint32_t>(max_threads_per_sm) / num_waves) / 32);
// Widths above 8 are non-portable and the query rejects them without this.
CHECK_CUDA(cudaFuncSetAttribute(
reinterpret_cast<const void*>(dummy_probe_kernel), cudaFuncAttributeNonPortableClusterSizeAllowed, 1));
CHECK_CUDA(cudaFuncSetAttribute(
reinterpret_cast<const void*>(dummy_probe_kernel),
cudaFuncAttributeMaxDynamicSharedMemorySize,
static_cast<int>(smem)));
cudaLaunchConfig_t config = {}; // stream/dynamicSmemBytes must not be garbage
config.gridDim = dim3{cluster_size, 1024u};
config.blockDim = dim3{32, num_warps};
const auto kernel = reinterpret_cast<const void*>(dummy_probe_kernel);
if (cluster_size > 8) {
// Widths above 8 are non-portable and the query rejects them without this.
CHECK_CUDA(cudaFuncSetAttribute(kernel, cudaFuncAttributeNonPortableClusterSizeAllowed, 1));
}
CHECK_CUDA(cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, smem));
int probed_occupancy = 0;
const auto block_size = 32 * num_warps;
CHECK_CUDA(cudaOccupancyMaxActiveBlocksPerMultiprocessor(&probed_occupancy, dummy_probe_kernel, block_size, smem));
CHECK_HOST(probed_occupancy == static_cast<int>(num_waves));
cudaLaunchConfig_t config = {};
config.gridDim = dim3{cluster_size};
config.blockDim = dim3{block_size};
config.dynamicSmemBytes = smem;
config.numAttrs = 1;
cudaLaunchAttribute attr = {};
attr.id = cudaLaunchAttributeClusterDimension;
attr.val.clusterDim = {cluster_size, 1, 1};
config.attrs = &attr;
CHECK_CUDA(cudaOccupancyMaxActiveClusters(&num_clusters, dummy_probe_kernel, &config));
CHECK_CUDA(cudaOccupancyMaxActiveClusters(&num_clusters, kernel, &config));
return num_clusters;
#endif
}
@@ -21,6 +21,7 @@ from sglang.kernels.jit.utils.compile import (
cuda_stubs_dir,
load_jit,
make_cpp_args,
parallel_compile,
)
__all__ = [
@@ -34,6 +35,7 @@ __all__ = [
"is_musa_runtime",
"make_cpp_args",
"load_jit",
"parallel_compile",
"cuda_stubs_dir",
"override_jit_cuda_arch",
"get_activation_cuda_cflags",
@@ -13,6 +13,7 @@ Modules, in dependency order (there are no cycles)::
ninja generating and running build.ninja, reading its depfiles
cache build_key / deps_key, cache layout, publication
loader load_jit
parallel parallel_compile: several builds at once
"""
from sglang.kernels.jit.utils.compile.cpp_args import (
@@ -22,6 +23,7 @@ from sglang.kernels.jit.utils.compile.cpp_args import (
make_cpp_args,
)
from sglang.kernels.jit.utils.compile.loader import load_jit
from sglang.kernels.jit.utils.compile.parallel import parallel_compile
from sglang.kernels.jit.utils.compile.paths import (
DEFAULT_CFLAGS,
DEFAULT_INCLUDE,
@@ -43,4 +45,5 @@ __all__ = [
"cuda_stubs_dir",
"load_jit",
"make_cpp_args",
"parallel_compile",
]
@@ -0,0 +1,32 @@
"""``parallel_compile``: build several JIT modules at once."""
from __future__ import annotations
import os
from concurrent.futures import ThreadPoolExecutor
from typing import Callable, Iterable, List, Optional, TypeVar
T = TypeVar("T")
def parallel_compile(
builders: Iterable[Callable[[], T]],
max_workers: Optional[int] = None,
) -> List[T]:
"""Run module builders concurrently and return their results in order.
A builder is a call that ends in :func:`load_jit`, typically a ``cache_once``
module factory with its arguments bound. Each build is its own ninja/nvcc
subprocess under its own build lock, so the threads only wait on them, and a
cached module returns at once. Use it to build every variant a dispatch may
reach up front (e.g. before CUDA graph capture) instead of one at a time on
first use. Every builder runs to completion; the first exception is raised
after that.
"""
builders = list(builders)
if len(builders) <= 1:
return [build() for build in builders]
workers = max_workers or min(len(builders), (os.cpu_count() or 1) + 4)
with ThreadPoolExecutor(max_workers=workers, thread_name_prefix="jit-build") as ex:
futures = [ex.submit(build) for build in builders]
return [future.result() for future in futures]
@@ -1,5 +1,6 @@
from __future__ import annotations
import functools
from typing import TYPE_CHECKING, Dict, NamedTuple, Optional, Tuple
import torch
@@ -10,6 +11,7 @@ from sglang.kernels.jit.utils import (
cache_once,
load_jit,
make_cpp_args,
parallel_compile,
)
if TYPE_CHECKING:
@@ -19,7 +21,7 @@ if TYPE_CHECKING:
_INPUT_STRIDE_ALIGNMENT_BYTES = 128
_OUTPUT_STRIDE_ALIGNMENT_BYTES = 32
_CLUSTER_MAX_TOPK = 1024
_CLUSTER_MAX_BATCH_SIZE = 6
_CLUSTER_MIN_VOCAB_SIZE = 16384
class _Config(NamedTuple):
@@ -79,11 +81,26 @@ _NORMAL_CONFIGS: Dict[Tuple[torch.dtype, int], Tuple[_Config, ...]] = {
SUPPORTED_CUDA_ARCHS = ((9, 0), (10, 0), (10, 3))
# (max rows, cluster size), widest first: the most rows each cluster size runs in one wave
@cache_once
def _cluster_tuning(device_capability: Tuple[int, int]) -> Tuple[int, int]:
# Return (cluster_size, min_vocab_size)
major = device_capability[0]
return (8, 128 * 1024) if major == 9 else (16, 512 * 1024)
def _cluster_tiers() -> Tuple[Tuple[int, int], ...]:
from sglang.kernels.jit.utils.occupancy import (
NoSchedulableClustersError,
get_max_active_clusters,
)
tiers = []
# NOTE: only try these cluster sizes; add more variants if we need
for cluster_size in (16, 8, 4, 3, 2):
try:
# The cluster kernel runs one CTA per SM
max_rows = get_max_active_clusters(cluster_size, occupancy=1)
except NoSchedulableClustersError:
continue
# A narrower cluster earns a tier only by fitting more rows in one wave
if not tiers or max_rows > tiers[-1][0]:
tiers.append((max_rows, cluster_size))
return tuple(tiers)
@cache_once
@@ -120,7 +137,8 @@ def _jit_deep_select_module(
raise ValueError(
"the cluster kernel is tuned for bfloat16 at max_topk 1024 only"
)
configs = [_Config(max_topk, 256, 1, 4096, 4096, 16, cluster_size)]
# The cluster kernel streams its own stages; this tuning only fixes 512 threads
configs = [_Config(max_topk, 512, 1, 8192, 4096, 5, cluster_size)]
host_dispatch = "TopkCluster"
classes = [
config.to_cpp_class_name(
@@ -187,6 +205,51 @@ def is_deep_select_supported() -> bool:
return (major, minor) in SUPPORTED_CUDA_ARCHS
class _CachedModuleDispatch:
def __init__(self, mods: list[Module], rows: list[int], fallback: Module):
self._clusters = tuple(zip(rows, mods, strict=True))
self._fallback = fallback
def __call__(self, rows: int, vocab_size: int):
# NOTE: deficiency: cluster + sorted_index supports up to 1M context
if vocab_size > _CLUSTER_MIN_VOCAB_SIZE:
for max_rows, module in self._clusters:
if rows <= max_rows:
return module
return self._fallback
@cache_once
def _precompile(
*,
dtype: torch.dtype,
topk: int,
sorted: bool,
indices_type: torch.dtype,
sorted_index: bool,
return_value: bool,
page_transform: bool,
) -> _CachedModuleDispatch:
signature = (dtype, indices_type, sorted, sorted_index, return_value)
jobs = [(_get_max_topk_bucket(topk, False), 1)]
rows = []
if dtype == torch.bfloat16 and topk <= _CLUSTER_MAX_TOPK:
for max_rows, cluster_size in _cluster_tiers():
jobs.append((_get_max_topk_bucket(topk, True), cluster_size))
rows.append(max_rows)
modules = parallel_compile(
functools.partial(
_jit_deep_select_module,
*signature,
bucket,
cluster_size,
page_transform,
)
for bucket, cluster_size in jobs
)
return _CachedModuleDispatch(modules[1:], rows, modules[0])
def topk(
input: torch.Tensor,
topk: int,
@@ -201,8 +264,8 @@ def topk(
output_idx_offset: Optional[torch.Tensor] = None,
idx_oob_fill_value: int = 2147483647,
value_oob_fill_value: float = float("-inf"),
return_value: bool = True,
abort_when_nan_found: bool = True,
return_value: bool = False,
abort_when_nan_found: bool = False,
) -> Tuple[Optional[torch.Tensor], torch.Tensor]:
"""Select the largest ``topk`` values of every row of ``input``.
@@ -239,7 +302,7 @@ def topk_page_transform(
sorted_index: bool = False,
output_idx: Optional[torch.Tensor] = None,
idx_oob_fill_value: int = -1,
abort_when_nan_found: bool = True,
abort_when_nan_found: bool = False,
) -> torch.Tensor:
"""Top-k indices of every row of ``input``, written through a page table.
@@ -291,19 +354,21 @@ def _launch_topk_impl(
abort_when_nan_found: bool,
) -> Tuple[Optional[torch.Tensor], torch.Tensor]:
rows, vocab_size = input.shape
device_capability = torch.cuda.get_device_capability(input.device)
cluster_size, cluster_min_vocab_size = _cluster_tuning(device_capability)
use_cluster = (
input.dtype == torch.bfloat16
and rows <= _CLUSTER_MAX_BATCH_SIZE
and vocab_size >= cluster_min_vocab_size
and topk <= _CLUSTER_MAX_TOPK
dtype = input.dtype
dispatch = _precompile(
dtype=dtype,
topk=topk,
sorted=sorted,
indices_type=indices_type,
sorted_index=sorted_index,
return_value=return_value,
page_transform=page_table is not None,
)
values = None
if return_value:
values = aligned_new_empty(
(rows, topk),
input.dtype,
dtype,
input.device,
alignment_bytes=_OUTPUT_STRIDE_ALIGNMENT_BYTES,
)
@@ -314,20 +379,10 @@ def _launch_topk_impl(
input.device,
alignment_bytes=_OUTPUT_STRIDE_ALIGNMENT_BYTES,
)
module = _jit_deep_select_module(
input.dtype,
indices_type,
sorted,
sorted_index,
return_value,
_get_max_topk_bucket(topk, use_cluster),
cluster_size if use_cluster else 1,
page_table is not None,
)
input_storage_bytes = (
input.untyped_storage().nbytes() - input.storage_offset() * input.element_size()
)
module.topk(
dispatch(rows, vocab_size).topk(
input,
values,
output_idx,
@@ -80,8 +80,8 @@ def candidate_row_lens(
@cache_once
def _jit_block_amax_module():
args = make_cpp_args(is_arch_support_pdl())
def _jit_block_amax_module(in_dtype: torch.dtype, out_dtype: torch.dtype):
args = make_cpp_args(is_arch_support_pdl(), in_dtype, out_dtype)
return load_jit(
make_name("block_amax"),
*args,
@@ -105,15 +105,17 @@ def amax8_varlen(
selected anyway); ``topk=0`` never skips. ``out`` is allocated as
``[rows, ceil(max_seqlen / 8)]`` when not given, ``max_seqlen`` defaulting to
the width of ``scores``; every ``seq_lens[b]`` must fit in ``8 * out.shape[1]``.
fp32 only for now; ``scores`` rows must be 32-byte aligned (stride a multiple
of 8). Returns ``out``.
``scores`` and ``out`` are each fp32 or bf16 (``out`` defaults to fp32), the
rows of ``scores`` aligned to a block of 8 (stride a multiple of 8). Returns
``out``.
"""
if out is None:
num_tokens, max_len = scores.shape
if max_seqlen == 0:
max_seqlen = max_len
out = scores.new_empty(num_tokens, (max_seqlen + 7) // 8)
_jit_block_amax_module().amax8_varlen(scores, seq_lens, out, topk)
out = scores.new_empty(num_tokens, (max_seqlen + 7) // 8, dtype=torch.float32)
module = _jit_block_amax_module(scores.dtype, out.dtype)
module.amax8_varlen(scores, seq_lens, out, topk)
return out
@@ -136,7 +138,9 @@ def amax_topk_blocks(
plan = plan_topk_v2(nblocks)
# block maxima, the newest block +inf; the top-k reads each row up to nblocks
# only, so nothing past a row's keys is initialised (v2 needs stride % 4 == 0)
keys = logits.new_empty(rows, -(-max_seq_len // (4 * block)) * 4)
keys = logits.new_empty(
rows, -(-max_seq_len // (4 * block)) * 4, dtype=torch.float32
)
amax8_varlen(logits, seq_lens, out=keys)
blocks = torch.empty(rows, topk_blocks, dtype=torch.int32, device=logits.device)
topk_transform_paged_v2(keys, nblocks, None, blocks, 1, plan)
@@ -8,6 +8,7 @@ from typing import Iterator, Tuple
import torch
from sglang.srt.environ import envs
from sglang.srt.layers.attention.mqa_logits_utils import (
mqa_logits_row_bytes,
mqa_logits_rows_per_chunk,
@@ -16,16 +17,31 @@ from sglang.srt.utils.common import ceil_align
from .candidate_table import CANDIDATE_BLOCK_SIZE
_LOGITS_DTYPE = (
torch.bfloat16 if envs.SGLANG_OPT_DSV41_BF16_INDEX_LOGITS.get() else torch.float32
)
_LOGITS_DTYPE_BYTES = _LOGITS_DTYPE.itemsize
# Width alignment of the flat logits: covers block-wise reads of up to 64 columns, and 128 B of bf16, which
# DeepSelect reads past a row's end (DeepGEMM pads the stride but its tensor's storage ends at the last row)
FLAT_LOGITS_WIDTH_ALIGN = 64
def flat_index_logits_rows_per_tile(
rows: int, width: int, *, heads: int, budget_bytes: int
rows: int,
width: int,
*,
heads: int,
budget_bytes: int,
item_size: int = _LOGITS_DTYPE_BYTES,
) -> int:
"""Rows per fp32 logits tile within ``budget_bytes``, at the kernel's row
alignment."""
"""Rows per logits tile within ``budget_bytes`` (sized as fp32, which also
bounds bf16), at the kernel's row alignment."""
row_alignment = 128 // heads
# NOTE: for bf16, the estimated row bytes is 2 times the actual bytes
effective_width = width * item_size // 4
rows_per_chunk = mqa_logits_rows_per_chunk(
num_rows=ceil_align(rows, row_alignment),
row_bytes=mqa_logits_row_bytes(width),
row_bytes=mqa_logits_row_bytes(effective_width),
budget_bytes=budget_bytes,
)
if rows_per_chunk is None:
@@ -42,16 +58,15 @@ def flat_index_logits_tiles(
lengths: torch.Tensor,
context_lengths: list[int],
budget_bytes: int,
width_align: int = 4,
) -> Iterator[tuple[slice, torch.Tensor]]:
"""``(rows, logits)`` per row tile, each fp32 tile within ``budget_bytes``:
"""``(rows, logits)`` per row tile, each tile within ``budget_bytes``:
``logits[i, j]`` scores query row ``rows.start + i`` against ``kv[starts + j]``,
garbage past the row's ``lengths``; the width is ``max(context_lengths)``
aligned to ``width_align``."""
aligned to ``FLAT_LOGITS_WIDTH_ALIGN``."""
from deep_gemm import fp8_fp4_mqa_logits
rows = q[0].shape[0]
width = ceil_align(max(context_lengths, default=0), width_align)
width = ceil_align(max(context_lengths, default=0), FLAT_LOGITS_WIDTH_ALIGN)
if rows == 0 or width == 0:
return
rows_per_chunk = flat_index_logits_rows_per_tile(
@@ -70,6 +85,7 @@ def flat_index_logits_tiles(
tile_starts + lengths[tile],
False,
width,
logits_dtype=_LOGITS_DTYPE,
),
)
@@ -98,6 +114,7 @@ def deep_gemm_fp4_paged_mqa_logits(
deep_gemm_metadata,
max_seq_len,
False,
logits_dtype=_LOGITS_DTYPE,
)
+3
View File
@@ -1631,6 +1631,9 @@ class Envs:
# Run the DeepSeek-V4.1 ratio-1/2 prefill indexer on the torch path instead
# of the DeepGEMM dense fp4 logits kernel (test oracle / fallback).
SGLANG_DSV41_TORCH_PREFILL_INDEXER = EnvBool(False)
# DeepSeek-V4.1 ratio-1/2 index layers: the DeepGEMM fp4 logits kernels store bf16 instead
# of fp32 (the reference ranks bf16 scores); every consumer of those logits must take bf16.
SGLANG_OPT_DSV41_BF16_INDEX_LOGITS = EnvBool(False)
SGLANG_FP8_PAGED_MQA_LOGITS_TORCH = EnvBool(False)
SGLANG_OPT_FLASHMLA_SPARSE_PREFILL = EnvBool(True)
# gfx950 DeepSeek-V4.1 prefill: attend with aiter's OPUS sparse kernel over a bf16
@@ -443,6 +443,26 @@ def topk_transform_flashinfer_fused(
)
def topk_transform_paged_bf16(
logits: torch.Tensor,
seq_lens: torch.Tensor,
page_table: torch.Tensor,
out_page_indices: torch.Tensor,
page_size: int,
) -> None:
"""DeepSelect paged top-k of bf16 logits: pool slots only, unordered."""
from sglang.kernels.ops.attention.deep_select import topk_page_transform
topk_page_transform(
logits,
out_page_indices.shape[1],
page_table=page_table,
page_size=page_size,
end=seq_lens.reshape(-1).to(torch.int32),
output_idx=out_page_indices,
)
def topk_transform_paged_from_metadata(
logits: torch.Tensor,
metadata,
@@ -464,6 +484,16 @@ def topk_transform_paged_from_metadata(
page_table = metadata.page_table[rows]
out_page_indices = page_indices[rows]
out_raw_indices = raw_indices[rows] if raw_indices is not None else None
if logits.dtype == torch.bfloat16:
assert out_raw_indices is None
topk_transform_paged_bf16(
logits,
seq_lens,
page_table,
out_page_indices,
metadata.compressed_page_size,
)
return
if metadata.use_topk_v2:
topk_transform_paged_v2(
logits,
@@ -14,6 +14,7 @@ from sglang.kernels.ops.attention.dsv4.candidate_blocks import (
select_candidate_block_ids,
topk_among_blocks,
)
from sglang.kernels.ops.attention.dsv4.index_logits import FLAT_LOGITS_WIDTH_ALIGN
from .scoring import (
DeepGEMMPrefillData,
@@ -23,6 +24,7 @@ from .scoring import (
prefill_requests,
score_tiles,
select_decode,
topk_ragged,
write_decode,
write_prefill,
)
@@ -246,14 +248,12 @@ def _publish_prefill_blocks(
topk_blocks: int,
block_size: int,
) -> Tuple[torch.Tensor, torch.Tensor]:
from sglang.kernels.ops.attention.dsv4 import topk_transform_ragged_v2
selected = data.empty_selection(topk)
width = min(topk_blocks, -(-max(data.lens_per_request, default=0) // block_size))
blocks = torch.full(
(data.num_rows, width), -1, dtype=torch.int32, device=selected.device
)
for tile, logits in score_tiles(data, kv, width_align=4):
for tile, logits in score_tiles(data, kv):
_publish_tile_blocks(
data=data,
tile=tile,
@@ -262,11 +262,11 @@ def _publish_prefill_blocks(
topk_blocks=topk_blocks,
block_size=block_size,
)
topk_transform_ragged_v2(
topk_ragged(
logits,
data.compress_lens[tile],
out_offsets=data.request_starts[tile],
out_indices=selected[tile],
out=selected[tile],
offsets=data.request_starts[tile],
)
# Free this tile's logits before the generator scores the next one.
del logits
@@ -307,7 +307,9 @@ def _consume_prefill_blocks(
block_size: int,
) -> torch.Tensor:
selected = data.empty_selection(topk)
for tile, logits in score_tiles(data, kv, width_align=block_size):
# the block-aligned slices of `_consume_tile_blocks` stay within the logits width
assert FLAT_LOGITS_WIDTH_ALIGN % block_size == 0
for tile, logits in score_tiles(data, kv):
_consume_tile_blocks(
data=data,
tile=tile,
@@ -5,7 +5,7 @@ a selection back into pool slots."""
from __future__ import annotations
import itertools
from typing import TYPE_CHECKING, Generator, Iterator, List, Optional, Tuple
from typing import TYPE_CHECKING, Iterator, List, Optional, Tuple
import msgspec
import torch
@@ -207,12 +207,7 @@ def get_deep_gemm_prefill_data(
)
def score_tiles(
data: DeepGEMMPrefillData,
kv: Tuple[torch.Tensor, torch.Tensor],
*,
width_align: int,
) -> Generator[Tuple[slice, torch.Tensor], None, None]:
def score_tiles(data: DeepGEMMPrefillData, kv: Tuple[torch.Tensor, torch.Tensor]):
yield from flat_index_logits_tiles(
q=(data.q_fp4, data.q_sf),
kv=kv,
@@ -221,10 +216,35 @@ def score_tiles(
lengths=data.compress_lens,
context_lengths=data.lens_per_request,
budget_bytes=_DEEP_GEMM_SCORE_BUDGET_BYTES,
width_align=width_align,
)
def topk_ragged(
logits: torch.Tensor,
lens: torch.Tensor,
*,
out: torch.Tensor,
offsets: torch.Tensor,
) -> None:
"""Top-k of each row's first ``lens`` columns plus ``offsets`` into ``out``,
``-1`` padded, unordered: DeepSelect for bf16 logits, top-k v2 for fp32."""
if logits.dtype == torch.bfloat16:
from sglang.kernels.ops.attention.deep_select import topk
topk(
logits,
out.shape[1],
end=lens,
output_idx=out,
output_idx_offset=offsets,
idx_oob_fill_value=-1,
)
return
from sglang.kernels.ops.attention.dsv4 import topk_transform_ragged_v2
topk_transform_ragged_v2(logits, lens, out_offsets=offsets, out_indices=out)
def dense_prefill_topk(
data: DeepGEMMPrefillData,
kv: Tuple[torch.Tensor, torch.Tensor],
@@ -232,16 +252,11 @@ def dense_prefill_topk(
out: torch.Tensor,
) -> None:
"""Request-local compressed positions into ``out``, ``-1`` padded, unordered."""
from sglang.kernels.ops.attention.dsv4 import topk_transform_ragged_v2
# Score columns are request-relative, so a zero offset emits request-local positions.
zero_offsets = torch.zeros_like(data.request_starts)
for tile, logits in score_tiles(data, kv, width_align=4):
topk_transform_ragged_v2(
logits,
data.compress_lens[tile],
out_offsets=zero_offsets[tile],
out_indices=out[tile],
for tile, logits in score_tiles(data, kv):
topk_ragged(
logits, data.compress_lens[tile], out=out[tile], offsets=zero_offsets[tile]
)
# Free this tile's logits before the generator scores the next one.
del logits
@@ -28,7 +28,10 @@ from sglang.kernels.ops.attention.dsv4.topk import (
topk_transform_ragged_v2,
topk_transform_sparse,
)
from sglang.srt.layers.attention.dsv4.indexer import topk_transform_paged_from_metadata
from sglang.srt.layers.attention.dsv4.indexer import (
topk_transform_paged_bf16,
topk_transform_paged_from_metadata,
)
from sglang.srt.layers.attention.dsv4.metadata import expand_index_page_table
from sglang.srt.utils.common import async_h2d
@@ -39,6 +42,7 @@ from .scoring import (
get_deep_gemm_prefill_data,
get_index_k_cache,
score_tiles,
topk_ragged,
)
from .types import (
CandidateMetadata,
@@ -188,14 +192,23 @@ class SparseTableBackend:
# The block-selection chain reads logits after the main stream moves on.
logits.record_stream(self.alt_stream)
# TODO(candidate): one kernel for both selections below (dense logits read once)
topk_transform_paged_v2(
logits,
seq_lens,
metadata.page_table,
inputs.out_page_indices,
metadata.compressed_page_size,
metadata.topk_metadata,
)
if logits.dtype == torch.bfloat16:
topk_transform_paged_bf16(
logits,
seq_lens,
metadata.page_table,
inputs.out_page_indices,
metadata.compressed_page_size,
)
else:
topk_transform_paged_v2(
logits,
seq_lens,
metadata.page_table,
inputs.out_page_indices,
metadata.compressed_page_size,
metadata.topk_metadata,
)
# per row: block count for the block top-k, sparse-row length for the consumers
with torch.cuda.stream(self.alt_stream):
nblocks, row_valid_lens = candidate_row_lens(seq_lens, self.topk_blocks)
@@ -403,20 +416,17 @@ def publish_prefill_table(
blocks = torch.empty(rows, topk_blocks, dtype=torch.int32, device=device)
zero_offsets = torch.zeros(rows, dtype=torch.int32, device=device)
# the block keys read the score rows through 32-byte vectors
for tile, logits in score_tiles(data, kv, width_align=8):
for tile, logits in score_tiles(data, kv):
lens = data.compress_lens[tile]
topk_transform_ragged_v2(
logits,
lens,
out_offsets=zero_offsets[tile],
out_indices=out_positions[tile],
)
topk_ragged(logits, lens, out=out_positions[tile], offsets=zero_offsets[tile])
# keys past a row's block count stay unset; the top-k reads a row
# up to its block count only (v2 wants the stride a multiple of 4)
width_blocks = (
logits.shape[1] + CANDIDATE_BLOCK_SIZE - 1
) // CANDIDATE_BLOCK_SIZE
keys = logits.new_empty(logits.shape[0], (width_blocks + 3) // 4 * 4)
keys = logits.new_empty(
logits.shape[0], (width_blocks + 3) // 4 * 4, dtype=torch.float32
)
amax8_varlen(logits, lens, out=keys)
topk_transform_ragged_v2(
keys,