[DeepSeek-V4.1] DeepSelect cluster top-k for long rows and bf16 index logits (#44051)
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Co-authored-by: Xiaoyu Zhang <1182563586@qq.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
Xiaoyu Zhang
parent
c5c4bbbd25
commit
9d68ecce2e
+20
-6
@@ -649,9 +649,10 @@ public:
|
||||
// Save result to `smem.reconstruct_pivot_bucket` and `smem.reconstruct_num_should_select`
|
||||
// If `CHECK_IF_SHOULD_SELECT_WHOLE_BUCKET`, return whether we should select all elements in that bucket.
|
||||
// Should be called by a whole warp
|
||||
template<bool CHECK_IF_SHOULD_SELECT_WHOLE_BUCKET>
|
||||
// [SGLang] Any plan with `reconstruct_pivot_bucket` and `reconstruct_num_should_select` will do
|
||||
template<bool CHECK_IF_SHOULD_SELECT_WHOLE_BUCKET, typename PlanT>
|
||||
static __device__ __forceinline__
|
||||
bool find_pivot_in_histogram(SharedMemoryPlanBase &smem, const uint32_t *bucket_counter, uint32_t topk, uint32_t lane_idx) {
|
||||
bool find_pivot_in_histogram(PlanT &smem, const uint32_t *bucket_counter, uint32_t topk, uint32_t lane_idx) {
|
||||
bool should_select_whole_bucket = false;
|
||||
// Each lane loads 8 counters
|
||||
static_assert(NUM_RECONSTRUCT_BUCKETS == 32 * 8);
|
||||
@@ -954,8 +955,10 @@ public:
|
||||
);
|
||||
}
|
||||
|
||||
// Take the appropriate action when we see a NaN (i.e. `trap()` or write 0x3f3f3f3f)
|
||||
static __device__ __forceinline__
|
||||
// Take the appropriate action when we see a NaN: `trap()`, or [SGLang] write the row as if every selected
|
||||
// index were 0, so downstream gathers stay in range. Called by the whole CTA; out of line, since inlined copies
|
||||
// of this cold path only cost instruction fetch on every launch.
|
||||
static __device__ __noinline__
|
||||
void take_action_when_have_nan(const TopkSelectArgs &args, uint32_t batch_idx) {
|
||||
if (args.abort_when_nan_found) {
|
||||
if (threadIdx.x == 0) {
|
||||
@@ -964,8 +967,19 @@ public:
|
||||
__syncthreads();
|
||||
ku::trap();
|
||||
} else {
|
||||
if (threadIdx.x == 0) {
|
||||
*((OutIdxT*)args.output_index + batch_idx * args.stride_output_index_batch) = 0x3F3F3F3F;
|
||||
OutIdxT* result_indices = (OutIdxT*)args.output_index + (uint64_t)batch_idx * args.stride_output_index_batch;
|
||||
OutIdxT index0;
|
||||
if constexpr (Config::page_transform) {
|
||||
const int* page_table_row = args.page_table + (uint64_t)batch_idx * args.stride_page_table_batch;
|
||||
index0 = (OutIdxT)EpilogueT::page_transform_index(0u, true, 0, page_table_row, args.page_bits);
|
||||
} else {
|
||||
index0 = (OutIdxT)(args.output_idx_offset != nullptr ? __ldg(args.output_idx_offset + batch_idx) : 0);
|
||||
}
|
||||
for (uint32_t i = threadIdx.x; i < args.topk; i += NUM_THREADS) result_indices[i] = index0;
|
||||
if constexpr (Config::return_value) {
|
||||
const ValueT value0 = ((const ValueT*)args.input)[(uint64_t)batch_idx * args.stride_input_batch];
|
||||
ValueT* result_values = (ValueT*)args.output_value + (uint64_t)batch_idx * args.stride_output_value_batch;
|
||||
for (uint32_t i = threadIdx.x; i < args.topk; i += NUM_THREADS) result_values[i] = value0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+887
-205
File diff suppressed because it is too large
Load Diff
@@ -9,30 +9,39 @@
|
||||
#include <dlpack/dlpack.h>
|
||||
#include <tvm/ffi/container/tensor.h>
|
||||
|
||||
#include <algorithm>
|
||||
#include <cstdint>
|
||||
#include <limits>
|
||||
#include <type_traits>
|
||||
|
||||
namespace sglang {
|
||||
|
||||
/// Level-one keys of the two-level indexer: the max of each kBlockTokens-score
|
||||
/// block, the row's newest block forced to +inf. Contract: BlockAmaxKernel.
|
||||
/// block, the row's newest block forced to +inf. Scores and keys are each fp32
|
||||
/// or bf16; a bf16 score's max is exact in either key type. Contract: BlockAmaxKernel.
|
||||
template <typename InT, typename OutT>
|
||||
struct BlockAmaxConfig {
|
||||
using DType = float;
|
||||
using DType = InT;
|
||||
using KeyT = OutT;
|
||||
static constexpr uint32_t kBlockTokens = 8; // scores per key
|
||||
static constexpr uint32_t kBlockSize = 512;
|
||||
static constexpr uint32_t kNumItems = 2; // keys per thread
|
||||
static constexpr uint32_t kOccupancy = 4;
|
||||
static constexpr uint32_t kKeysPerCTA = kBlockSize * kNumItems;
|
||||
// One block is 32 B: a single load on Blackwell, two 16 B loads before it.
|
||||
static constexpr uint32_t kVecSize = device::kMaxVecBytes / sizeof(DType);
|
||||
// One fp32 block is 32 B: a single load on Blackwell, two 16 B loads before it.
|
||||
// A bf16 block is one 16 B load anywhere.
|
||||
static constexpr uint32_t kBlockBytes = kBlockTokens * sizeof(DType);
|
||||
static constexpr uint32_t kVecBytes = std::min<uint32_t>(kBlockBytes, device::kMaxVecBytes);
|
||||
static constexpr uint32_t kVecSize = kVecBytes / sizeof(DType);
|
||||
static constexpr uint32_t kVecsPerBlock = kBlockTokens / kVecSize;
|
||||
static constexpr uint32_t kNumItems = 64 / kBlockBytes; // 64B scores (16 registers) per thread
|
||||
static constexpr uint32_t kKeysPerCTA = kBlockSize * kNumItems;
|
||||
static_assert(kVecsPerBlock * kVecSize == kBlockTokens);
|
||||
using vec_t = device::AlignedVector<DType, kVecSize>;
|
||||
};
|
||||
|
||||
template <typename InT, typename OutT>
|
||||
struct BlockAmaxParams {
|
||||
const BlockAmaxConfig::DType* __restrict__ scores;
|
||||
BlockAmaxConfig::DType* __restrict__ amax_scores;
|
||||
const InT* __restrict__ scores;
|
||||
OutT* __restrict__ amax_scores;
|
||||
const int32_t* __restrict__ seq_len;
|
||||
int64_t stride_scores; // in elements
|
||||
int64_t stride_amax_scores; // in elements
|
||||
@@ -41,12 +50,11 @@ struct BlockAmaxParams {
|
||||
|
||||
/// grid = (rows, ceil(max_keys / kKeysPerCTA)); a CTA owns kKeysPerCTA consecutive
|
||||
/// keys of one row, a thread kNumItems keys kBlockSize apart (coalesced loads).
|
||||
template <bool kUsePDL>
|
||||
__global__ __launch_bounds__(BlockAmaxConfig::kBlockSize, BlockAmaxConfig::kOccupancy) //
|
||||
void amax8_varlen_kernel(const __grid_constant__ BlockAmaxParams params) {
|
||||
template <bool kUsePDL, typename InT, typename OutT>
|
||||
__global__ __launch_bounds__(BlockAmaxConfig<InT, OutT>::kBlockSize, BlockAmaxConfig<InT, OutT>::kOccupancy) //
|
||||
void amax8_varlen_kernel(const __grid_constant__ BlockAmaxParams<InT, OutT> params) {
|
||||
using namespace device;
|
||||
using C = BlockAmaxConfig;
|
||||
using T = typename C::DType;
|
||||
using C = BlockAmaxConfig<InT, OutT>;
|
||||
using vec_t = typename C::vec_t;
|
||||
const auto bx = blockIdx.x;
|
||||
const auto by = blockIdx.y;
|
||||
@@ -81,25 +89,41 @@ __global__ __launch_bounds__(BlockAmaxConfig::kBlockSize, BlockAmaxConfig::kOccu
|
||||
for (uint32_t i = 0; i < C::kNumItems; ++i) {
|
||||
const auto idx = first_key + tx + i * C::kBlockSize;
|
||||
if (idx < num_keys) {
|
||||
T key = vec[i][0][0];
|
||||
// fmaxf and __hmax2 ignore a NaN score, torch.amax would propagate it
|
||||
if constexpr (std::is_same_v<InT, float>) {
|
||||
float key = vec[i][0][0];
|
||||
#pragma unroll
|
||||
for (uint32_t v = 0; v < C::kVecsPerBlock; ++v) {
|
||||
for (uint32_t v = 0; v < C::kVecsPerBlock; ++v) {
|
||||
#pragma unroll
|
||||
for (uint32_t j = 0; j < C::kVecSize; ++j) {
|
||||
key = fmaxf(key, vec[i][v][j]); // a NaN score is ignored, torch.amax would propagate it
|
||||
for (uint32_t j = 0; j < C::kVecSize; ++j) {
|
||||
key = fmaxf(key, vec[i][v][j]);
|
||||
}
|
||||
}
|
||||
const auto fp32_inf = std::numeric_limits<float>::infinity();
|
||||
out[idx] = cast<OutT>(idx + 1 == num_keys ? fp32_inf : key);
|
||||
} else {
|
||||
const auto vec_ptr = reinterpret_cast<bf16x2_t*>(vec[i][0].data());
|
||||
static_assert(C::kVecSize % 2 == 0);
|
||||
bf16x2_t key2 = vec_ptr[0];
|
||||
#pragma unroll
|
||||
for (uint32_t v = 1; v < C::kVecsPerBlock * C::kVecSize / 2; ++v) {
|
||||
key2 = __hmax2(key2, vec_ptr[v]);
|
||||
}
|
||||
const auto key = __hmax(key2.x, key2.y);
|
||||
const auto bf16_inf = cast<bf16_t>(std::numeric_limits<float>::infinity());
|
||||
out[idx] = cast<OutT>(idx + 1 == num_keys ? bf16_inf : key);
|
||||
}
|
||||
out[idx] = idx + 1 == num_keys ? std::numeric_limits<T>::infinity() : key;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Host entry: `amax_scores[b, i] = max(scores[b, 8 i : 8 i + 8])` for
|
||||
/// `i < ceil(seq_len[b] / 8)`, the last of them +inf; rows with at most `topk`
|
||||
/// blocks untouched. `scores` rows must stay 32 B aligned (stride % 8 == 0).
|
||||
/// blocks untouched. `scores` (`InT`) rows must keep every block of 8 aligned
|
||||
/// (stride % 8 == 0); `amax_scores` is `OutT`. Each type is fp32 or bf16.
|
||||
/// The grid covers `amax_scores`' width, so the caller sizes it for the longest
|
||||
/// row: `seq_len[b] <= 8 * amax_scores.shape[1]` for every row (not checked).
|
||||
template <bool kPDL>
|
||||
template <bool kPDL, typename InT, typename OutT>
|
||||
struct BlockAmaxKernel {
|
||||
static void amax8_varlen(
|
||||
const tvm::ffi::TensorView scores,
|
||||
@@ -107,40 +131,34 @@ struct BlockAmaxKernel {
|
||||
const tvm::ffi::TensorView amax_scores,
|
||||
const uint32_t topk) {
|
||||
using namespace host;
|
||||
using C = BlockAmaxConfig;
|
||||
using C = BlockAmaxConfig<InT, OutT>;
|
||||
auto B = SymbolicSize{"batch_size"};
|
||||
auto L = SymbolicSize{"max_seq_len"};
|
||||
auto S = SymbolicSize{"stride_scores"};
|
||||
auto K = SymbolicSize{"max_keys"};
|
||||
auto O = SymbolicSize{"stride_amax_scores"};
|
||||
auto device_ = SymbolicDevice{};
|
||||
device_.set_options<kDLGPU>();
|
||||
TensorMatcher({B, L}) // scores
|
||||
.with_strides({S, 1})
|
||||
.with_strides({-1, 1})
|
||||
.with_dtype<typename C::DType>()
|
||||
.with_device(device_)
|
||||
.ensure_alignment(C::kBlockBytes)
|
||||
.verify(scores);
|
||||
TensorMatcher({B}) // seq_lens
|
||||
.with_dtype<int32_t>()
|
||||
.with_device(device_)
|
||||
.verify(seq_lens);
|
||||
TensorMatcher({B, K}) // amax_scores
|
||||
.with_strides({O, 1})
|
||||
.with_dtype<typename C::DType>()
|
||||
.with_strides({-1, 1})
|
||||
.with_dtype<typename C::KeyT>()
|
||||
.with_device(device_)
|
||||
.verify(amax_scores);
|
||||
RuntimeCheck(S.unwrap() % C::kBlockTokens == 0, "stride_scores must keep every block 32 B aligned");
|
||||
RuntimeCheck(
|
||||
reinterpret_cast<uintptr_t>(scores.data_ptr()) % (C::kBlockTokens * sizeof(typename C::DType)) == 0,
|
||||
"scores must be 32 B aligned");
|
||||
RuntimeCheck(K.unwrap() > 0, "amax_scores must hold at least one key per row");
|
||||
const auto max_keys = K.unwrap(); // ceil(longest row / 8), sized by the caller
|
||||
const auto params = BlockAmaxParams{
|
||||
.scores = static_cast<const typename C::DType*>(scores.data_ptr()),
|
||||
.amax_scores = static_cast<typename C::DType*>(amax_scores.data_ptr()),
|
||||
const auto params = BlockAmaxParams<InT, OutT>{
|
||||
.scores = static_cast<const InT*>(scores.data_ptr()),
|
||||
.amax_scores = static_cast<OutT*>(amax_scores.data_ptr()),
|
||||
.seq_len = static_cast<const int32_t*>(seq_lens.data_ptr()),
|
||||
.stride_scores = S.unwrap(),
|
||||
.stride_amax_scores = O.unwrap(),
|
||||
.stride_scores = scores.stride(0),
|
||||
.stride_amax_scores = amax_scores.stride(0),
|
||||
.topk = topk,
|
||||
};
|
||||
const auto grid = dim3(
|
||||
@@ -148,7 +166,7 @@ struct BlockAmaxKernel {
|
||||
static_cast<uint32_t>(div_ceil(max_keys, static_cast<int64_t>(C::kKeysPerCTA))));
|
||||
LaunchKernel(grid, C::kBlockSize, device_.unwrap())
|
||||
.config({.use_pdl = kPDL})
|
||||
.launch(amax8_varlen_kernel<kPDL>, params);
|
||||
.launch(amax8_varlen_kernel<kPDL, InT, OutT>, params);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@@ -30,24 +30,26 @@ uint32_t get_max_active_clusters(uint32_t cluster_size, uint32_t num_waves) {
|
||||
const auto smem = (std::min(budget - std::min(budget, reserved), static_cast<uint32_t>(smem_per_block))) & ~1023u;
|
||||
const auto num_warps = std::max(1u, std::min(1024u, static_cast<uint32_t>(max_threads_per_sm) / num_waves) / 32);
|
||||
|
||||
// Widths above 8 are non-portable and the query rejects them without this.
|
||||
CHECK_CUDA(cudaFuncSetAttribute(
|
||||
reinterpret_cast<const void*>(dummy_probe_kernel), cudaFuncAttributeNonPortableClusterSizeAllowed, 1));
|
||||
CHECK_CUDA(cudaFuncSetAttribute(
|
||||
reinterpret_cast<const void*>(dummy_probe_kernel),
|
||||
cudaFuncAttributeMaxDynamicSharedMemorySize,
|
||||
static_cast<int>(smem)));
|
||||
|
||||
cudaLaunchConfig_t config = {}; // stream/dynamicSmemBytes must not be garbage
|
||||
config.gridDim = dim3{cluster_size, 1024u};
|
||||
config.blockDim = dim3{32, num_warps};
|
||||
const auto kernel = reinterpret_cast<const void*>(dummy_probe_kernel);
|
||||
if (cluster_size > 8) {
|
||||
// Widths above 8 are non-portable and the query rejects them without this.
|
||||
CHECK_CUDA(cudaFuncSetAttribute(kernel, cudaFuncAttributeNonPortableClusterSizeAllowed, 1));
|
||||
}
|
||||
CHECK_CUDA(cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, smem));
|
||||
int probed_occupancy = 0;
|
||||
const auto block_size = 32 * num_warps;
|
||||
CHECK_CUDA(cudaOccupancyMaxActiveBlocksPerMultiprocessor(&probed_occupancy, dummy_probe_kernel, block_size, smem));
|
||||
CHECK_HOST(probed_occupancy == static_cast<int>(num_waves));
|
||||
cudaLaunchConfig_t config = {};
|
||||
config.gridDim = dim3{cluster_size};
|
||||
config.blockDim = dim3{block_size};
|
||||
config.dynamicSmemBytes = smem;
|
||||
config.numAttrs = 1;
|
||||
cudaLaunchAttribute attr = {};
|
||||
attr.id = cudaLaunchAttributeClusterDimension;
|
||||
attr.val.clusterDim = {cluster_size, 1, 1};
|
||||
config.attrs = &attr;
|
||||
CHECK_CUDA(cudaOccupancyMaxActiveClusters(&num_clusters, dummy_probe_kernel, &config));
|
||||
CHECK_CUDA(cudaOccupancyMaxActiveClusters(&num_clusters, kernel, &config));
|
||||
return num_clusters;
|
||||
#endif
|
||||
}
|
||||
|
||||
@@ -21,6 +21,7 @@ from sglang.kernels.jit.utils.compile import (
|
||||
cuda_stubs_dir,
|
||||
load_jit,
|
||||
make_cpp_args,
|
||||
parallel_compile,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
@@ -34,6 +35,7 @@ __all__ = [
|
||||
"is_musa_runtime",
|
||||
"make_cpp_args",
|
||||
"load_jit",
|
||||
"parallel_compile",
|
||||
"cuda_stubs_dir",
|
||||
"override_jit_cuda_arch",
|
||||
"get_activation_cuda_cflags",
|
||||
|
||||
@@ -13,6 +13,7 @@ Modules, in dependency order (there are no cycles)::
|
||||
ninja generating and running build.ninja, reading its depfiles
|
||||
cache build_key / deps_key, cache layout, publication
|
||||
loader load_jit
|
||||
parallel parallel_compile: several builds at once
|
||||
"""
|
||||
|
||||
from sglang.kernels.jit.utils.compile.cpp_args import (
|
||||
@@ -22,6 +23,7 @@ from sglang.kernels.jit.utils.compile.cpp_args import (
|
||||
make_cpp_args,
|
||||
)
|
||||
from sglang.kernels.jit.utils.compile.loader import load_jit
|
||||
from sglang.kernels.jit.utils.compile.parallel import parallel_compile
|
||||
from sglang.kernels.jit.utils.compile.paths import (
|
||||
DEFAULT_CFLAGS,
|
||||
DEFAULT_INCLUDE,
|
||||
@@ -43,4 +45,5 @@ __all__ = [
|
||||
"cuda_stubs_dir",
|
||||
"load_jit",
|
||||
"make_cpp_args",
|
||||
"parallel_compile",
|
||||
]
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
"""``parallel_compile``: build several JIT modules at once."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from typing import Callable, Iterable, List, Optional, TypeVar
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
|
||||
def parallel_compile(
|
||||
builders: Iterable[Callable[[], T]],
|
||||
max_workers: Optional[int] = None,
|
||||
) -> List[T]:
|
||||
"""Run module builders concurrently and return their results in order.
|
||||
|
||||
A builder is a call that ends in :func:`load_jit`, typically a ``cache_once``
|
||||
module factory with its arguments bound. Each build is its own ninja/nvcc
|
||||
subprocess under its own build lock, so the threads only wait on them, and a
|
||||
cached module returns at once. Use it to build every variant a dispatch may
|
||||
reach up front (e.g. before CUDA graph capture) instead of one at a time on
|
||||
first use. Every builder runs to completion; the first exception is raised
|
||||
after that.
|
||||
"""
|
||||
builders = list(builders)
|
||||
if len(builders) <= 1:
|
||||
return [build() for build in builders]
|
||||
workers = max_workers or min(len(builders), (os.cpu_count() or 1) + 4)
|
||||
with ThreadPoolExecutor(max_workers=workers, thread_name_prefix="jit-build") as ex:
|
||||
futures = [ex.submit(build) for build in builders]
|
||||
return [future.result() for future in futures]
|
||||
@@ -1,5 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import functools
|
||||
from typing import TYPE_CHECKING, Dict, NamedTuple, Optional, Tuple
|
||||
|
||||
import torch
|
||||
@@ -10,6 +11,7 @@ from sglang.kernels.jit.utils import (
|
||||
cache_once,
|
||||
load_jit,
|
||||
make_cpp_args,
|
||||
parallel_compile,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -19,7 +21,7 @@ if TYPE_CHECKING:
|
||||
_INPUT_STRIDE_ALIGNMENT_BYTES = 128
|
||||
_OUTPUT_STRIDE_ALIGNMENT_BYTES = 32
|
||||
_CLUSTER_MAX_TOPK = 1024
|
||||
_CLUSTER_MAX_BATCH_SIZE = 6
|
||||
_CLUSTER_MIN_VOCAB_SIZE = 16384
|
||||
|
||||
|
||||
class _Config(NamedTuple):
|
||||
@@ -79,11 +81,26 @@ _NORMAL_CONFIGS: Dict[Tuple[torch.dtype, int], Tuple[_Config, ...]] = {
|
||||
SUPPORTED_CUDA_ARCHS = ((9, 0), (10, 0), (10, 3))
|
||||
|
||||
|
||||
# (max rows, cluster size), widest first: the most rows each cluster size runs in one wave
|
||||
@cache_once
|
||||
def _cluster_tuning(device_capability: Tuple[int, int]) -> Tuple[int, int]:
|
||||
# Return (cluster_size, min_vocab_size)
|
||||
major = device_capability[0]
|
||||
return (8, 128 * 1024) if major == 9 else (16, 512 * 1024)
|
||||
def _cluster_tiers() -> Tuple[Tuple[int, int], ...]:
|
||||
from sglang.kernels.jit.utils.occupancy import (
|
||||
NoSchedulableClustersError,
|
||||
get_max_active_clusters,
|
||||
)
|
||||
|
||||
tiers = []
|
||||
# NOTE: only try these cluster sizes; add more variants if we need
|
||||
for cluster_size in (16, 8, 4, 3, 2):
|
||||
try:
|
||||
# The cluster kernel runs one CTA per SM
|
||||
max_rows = get_max_active_clusters(cluster_size, occupancy=1)
|
||||
except NoSchedulableClustersError:
|
||||
continue
|
||||
# A narrower cluster earns a tier only by fitting more rows in one wave
|
||||
if not tiers or max_rows > tiers[-1][0]:
|
||||
tiers.append((max_rows, cluster_size))
|
||||
return tuple(tiers)
|
||||
|
||||
|
||||
@cache_once
|
||||
@@ -120,7 +137,8 @@ def _jit_deep_select_module(
|
||||
raise ValueError(
|
||||
"the cluster kernel is tuned for bfloat16 at max_topk 1024 only"
|
||||
)
|
||||
configs = [_Config(max_topk, 256, 1, 4096, 4096, 16, cluster_size)]
|
||||
# The cluster kernel streams its own stages; this tuning only fixes 512 threads
|
||||
configs = [_Config(max_topk, 512, 1, 8192, 4096, 5, cluster_size)]
|
||||
host_dispatch = "TopkCluster"
|
||||
classes = [
|
||||
config.to_cpp_class_name(
|
||||
@@ -187,6 +205,51 @@ def is_deep_select_supported() -> bool:
|
||||
return (major, minor) in SUPPORTED_CUDA_ARCHS
|
||||
|
||||
|
||||
class _CachedModuleDispatch:
|
||||
def __init__(self, mods: list[Module], rows: list[int], fallback: Module):
|
||||
self._clusters = tuple(zip(rows, mods, strict=True))
|
||||
self._fallback = fallback
|
||||
|
||||
def __call__(self, rows: int, vocab_size: int):
|
||||
# NOTE: deficiency: cluster + sorted_index supports up to 1M context
|
||||
if vocab_size > _CLUSTER_MIN_VOCAB_SIZE:
|
||||
for max_rows, module in self._clusters:
|
||||
if rows <= max_rows:
|
||||
return module
|
||||
return self._fallback
|
||||
|
||||
|
||||
@cache_once
|
||||
def _precompile(
|
||||
*,
|
||||
dtype: torch.dtype,
|
||||
topk: int,
|
||||
sorted: bool,
|
||||
indices_type: torch.dtype,
|
||||
sorted_index: bool,
|
||||
return_value: bool,
|
||||
page_transform: bool,
|
||||
) -> _CachedModuleDispatch:
|
||||
signature = (dtype, indices_type, sorted, sorted_index, return_value)
|
||||
jobs = [(_get_max_topk_bucket(topk, False), 1)]
|
||||
rows = []
|
||||
if dtype == torch.bfloat16 and topk <= _CLUSTER_MAX_TOPK:
|
||||
for max_rows, cluster_size in _cluster_tiers():
|
||||
jobs.append((_get_max_topk_bucket(topk, True), cluster_size))
|
||||
rows.append(max_rows)
|
||||
modules = parallel_compile(
|
||||
functools.partial(
|
||||
_jit_deep_select_module,
|
||||
*signature,
|
||||
bucket,
|
||||
cluster_size,
|
||||
page_transform,
|
||||
)
|
||||
for bucket, cluster_size in jobs
|
||||
)
|
||||
return _CachedModuleDispatch(modules[1:], rows, modules[0])
|
||||
|
||||
|
||||
def topk(
|
||||
input: torch.Tensor,
|
||||
topk: int,
|
||||
@@ -201,8 +264,8 @@ def topk(
|
||||
output_idx_offset: Optional[torch.Tensor] = None,
|
||||
idx_oob_fill_value: int = 2147483647,
|
||||
value_oob_fill_value: float = float("-inf"),
|
||||
return_value: bool = True,
|
||||
abort_when_nan_found: bool = True,
|
||||
return_value: bool = False,
|
||||
abort_when_nan_found: bool = False,
|
||||
) -> Tuple[Optional[torch.Tensor], torch.Tensor]:
|
||||
"""Select the largest ``topk`` values of every row of ``input``.
|
||||
|
||||
@@ -239,7 +302,7 @@ def topk_page_transform(
|
||||
sorted_index: bool = False,
|
||||
output_idx: Optional[torch.Tensor] = None,
|
||||
idx_oob_fill_value: int = -1,
|
||||
abort_when_nan_found: bool = True,
|
||||
abort_when_nan_found: bool = False,
|
||||
) -> torch.Tensor:
|
||||
"""Top-k indices of every row of ``input``, written through a page table.
|
||||
|
||||
@@ -291,19 +354,21 @@ def _launch_topk_impl(
|
||||
abort_when_nan_found: bool,
|
||||
) -> Tuple[Optional[torch.Tensor], torch.Tensor]:
|
||||
rows, vocab_size = input.shape
|
||||
device_capability = torch.cuda.get_device_capability(input.device)
|
||||
cluster_size, cluster_min_vocab_size = _cluster_tuning(device_capability)
|
||||
use_cluster = (
|
||||
input.dtype == torch.bfloat16
|
||||
and rows <= _CLUSTER_MAX_BATCH_SIZE
|
||||
and vocab_size >= cluster_min_vocab_size
|
||||
and topk <= _CLUSTER_MAX_TOPK
|
||||
dtype = input.dtype
|
||||
dispatch = _precompile(
|
||||
dtype=dtype,
|
||||
topk=topk,
|
||||
sorted=sorted,
|
||||
indices_type=indices_type,
|
||||
sorted_index=sorted_index,
|
||||
return_value=return_value,
|
||||
page_transform=page_table is not None,
|
||||
)
|
||||
values = None
|
||||
if return_value:
|
||||
values = aligned_new_empty(
|
||||
(rows, topk),
|
||||
input.dtype,
|
||||
dtype,
|
||||
input.device,
|
||||
alignment_bytes=_OUTPUT_STRIDE_ALIGNMENT_BYTES,
|
||||
)
|
||||
@@ -314,20 +379,10 @@ def _launch_topk_impl(
|
||||
input.device,
|
||||
alignment_bytes=_OUTPUT_STRIDE_ALIGNMENT_BYTES,
|
||||
)
|
||||
module = _jit_deep_select_module(
|
||||
input.dtype,
|
||||
indices_type,
|
||||
sorted,
|
||||
sorted_index,
|
||||
return_value,
|
||||
_get_max_topk_bucket(topk, use_cluster),
|
||||
cluster_size if use_cluster else 1,
|
||||
page_table is not None,
|
||||
)
|
||||
input_storage_bytes = (
|
||||
input.untyped_storage().nbytes() - input.storage_offset() * input.element_size()
|
||||
)
|
||||
module.topk(
|
||||
dispatch(rows, vocab_size).topk(
|
||||
input,
|
||||
values,
|
||||
output_idx,
|
||||
|
||||
@@ -80,8 +80,8 @@ def candidate_row_lens(
|
||||
|
||||
|
||||
@cache_once
|
||||
def _jit_block_amax_module():
|
||||
args = make_cpp_args(is_arch_support_pdl())
|
||||
def _jit_block_amax_module(in_dtype: torch.dtype, out_dtype: torch.dtype):
|
||||
args = make_cpp_args(is_arch_support_pdl(), in_dtype, out_dtype)
|
||||
return load_jit(
|
||||
make_name("block_amax"),
|
||||
*args,
|
||||
@@ -105,15 +105,17 @@ def amax8_varlen(
|
||||
selected anyway); ``topk=0`` never skips. ``out`` is allocated as
|
||||
``[rows, ceil(max_seqlen / 8)]`` when not given, ``max_seqlen`` defaulting to
|
||||
the width of ``scores``; every ``seq_lens[b]`` must fit in ``8 * out.shape[1]``.
|
||||
fp32 only for now; ``scores`` rows must be 32-byte aligned (stride a multiple
|
||||
of 8). Returns ``out``.
|
||||
``scores`` and ``out`` are each fp32 or bf16 (``out`` defaults to fp32), the
|
||||
rows of ``scores`` aligned to a block of 8 (stride a multiple of 8). Returns
|
||||
``out``.
|
||||
"""
|
||||
if out is None:
|
||||
num_tokens, max_len = scores.shape
|
||||
if max_seqlen == 0:
|
||||
max_seqlen = max_len
|
||||
out = scores.new_empty(num_tokens, (max_seqlen + 7) // 8)
|
||||
_jit_block_amax_module().amax8_varlen(scores, seq_lens, out, topk)
|
||||
out = scores.new_empty(num_tokens, (max_seqlen + 7) // 8, dtype=torch.float32)
|
||||
module = _jit_block_amax_module(scores.dtype, out.dtype)
|
||||
module.amax8_varlen(scores, seq_lens, out, topk)
|
||||
return out
|
||||
|
||||
|
||||
@@ -136,7 +138,9 @@ def amax_topk_blocks(
|
||||
plan = plan_topk_v2(nblocks)
|
||||
# block maxima, the newest block +inf; the top-k reads each row up to nblocks
|
||||
# only, so nothing past a row's keys is initialised (v2 needs stride % 4 == 0)
|
||||
keys = logits.new_empty(rows, -(-max_seq_len // (4 * block)) * 4)
|
||||
keys = logits.new_empty(
|
||||
rows, -(-max_seq_len // (4 * block)) * 4, dtype=torch.float32
|
||||
)
|
||||
amax8_varlen(logits, seq_lens, out=keys)
|
||||
blocks = torch.empty(rows, topk_blocks, dtype=torch.int32, device=logits.device)
|
||||
topk_transform_paged_v2(keys, nblocks, None, blocks, 1, plan)
|
||||
|
||||
@@ -8,6 +8,7 @@ from typing import Iterator, Tuple
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.attention.mqa_logits_utils import (
|
||||
mqa_logits_row_bytes,
|
||||
mqa_logits_rows_per_chunk,
|
||||
@@ -16,16 +17,31 @@ from sglang.srt.utils.common import ceil_align
|
||||
|
||||
from .candidate_table import CANDIDATE_BLOCK_SIZE
|
||||
|
||||
_LOGITS_DTYPE = (
|
||||
torch.bfloat16 if envs.SGLANG_OPT_DSV41_BF16_INDEX_LOGITS.get() else torch.float32
|
||||
)
|
||||
_LOGITS_DTYPE_BYTES = _LOGITS_DTYPE.itemsize
|
||||
# Width alignment of the flat logits: covers block-wise reads of up to 64 columns, and 128 B of bf16, which
|
||||
# DeepSelect reads past a row's end (DeepGEMM pads the stride but its tensor's storage ends at the last row)
|
||||
FLAT_LOGITS_WIDTH_ALIGN = 64
|
||||
|
||||
|
||||
def flat_index_logits_rows_per_tile(
|
||||
rows: int, width: int, *, heads: int, budget_bytes: int
|
||||
rows: int,
|
||||
width: int,
|
||||
*,
|
||||
heads: int,
|
||||
budget_bytes: int,
|
||||
item_size: int = _LOGITS_DTYPE_BYTES,
|
||||
) -> int:
|
||||
"""Rows per fp32 logits tile within ``budget_bytes``, at the kernel's row
|
||||
alignment."""
|
||||
"""Rows per logits tile within ``budget_bytes`` (sized as fp32, which also
|
||||
bounds bf16), at the kernel's row alignment."""
|
||||
row_alignment = 128 // heads
|
||||
# NOTE: for bf16, the estimated row bytes is 2 times the actual bytes
|
||||
effective_width = width * item_size // 4
|
||||
rows_per_chunk = mqa_logits_rows_per_chunk(
|
||||
num_rows=ceil_align(rows, row_alignment),
|
||||
row_bytes=mqa_logits_row_bytes(width),
|
||||
row_bytes=mqa_logits_row_bytes(effective_width),
|
||||
budget_bytes=budget_bytes,
|
||||
)
|
||||
if rows_per_chunk is None:
|
||||
@@ -42,16 +58,15 @@ def flat_index_logits_tiles(
|
||||
lengths: torch.Tensor,
|
||||
context_lengths: list[int],
|
||||
budget_bytes: int,
|
||||
width_align: int = 4,
|
||||
) -> Iterator[tuple[slice, torch.Tensor]]:
|
||||
"""``(rows, logits)`` per row tile, each fp32 tile within ``budget_bytes``:
|
||||
"""``(rows, logits)`` per row tile, each tile within ``budget_bytes``:
|
||||
``logits[i, j]`` scores query row ``rows.start + i`` against ``kv[starts + j]``,
|
||||
garbage past the row's ``lengths``; the width is ``max(context_lengths)``
|
||||
aligned to ``width_align``."""
|
||||
aligned to ``FLAT_LOGITS_WIDTH_ALIGN``."""
|
||||
from deep_gemm import fp8_fp4_mqa_logits
|
||||
|
||||
rows = q[0].shape[0]
|
||||
width = ceil_align(max(context_lengths, default=0), width_align)
|
||||
width = ceil_align(max(context_lengths, default=0), FLAT_LOGITS_WIDTH_ALIGN)
|
||||
if rows == 0 or width == 0:
|
||||
return
|
||||
rows_per_chunk = flat_index_logits_rows_per_tile(
|
||||
@@ -70,6 +85,7 @@ def flat_index_logits_tiles(
|
||||
tile_starts + lengths[tile],
|
||||
False,
|
||||
width,
|
||||
logits_dtype=_LOGITS_DTYPE,
|
||||
),
|
||||
)
|
||||
|
||||
@@ -98,6 +114,7 @@ def deep_gemm_fp4_paged_mqa_logits(
|
||||
deep_gemm_metadata,
|
||||
max_seq_len,
|
||||
False,
|
||||
logits_dtype=_LOGITS_DTYPE,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -1631,6 +1631,9 @@ class Envs:
|
||||
# Run the DeepSeek-V4.1 ratio-1/2 prefill indexer on the torch path instead
|
||||
# of the DeepGEMM dense fp4 logits kernel (test oracle / fallback).
|
||||
SGLANG_DSV41_TORCH_PREFILL_INDEXER = EnvBool(False)
|
||||
# DeepSeek-V4.1 ratio-1/2 index layers: the DeepGEMM fp4 logits kernels store bf16 instead
|
||||
# of fp32 (the reference ranks bf16 scores); every consumer of those logits must take bf16.
|
||||
SGLANG_OPT_DSV41_BF16_INDEX_LOGITS = EnvBool(False)
|
||||
SGLANG_FP8_PAGED_MQA_LOGITS_TORCH = EnvBool(False)
|
||||
SGLANG_OPT_FLASHMLA_SPARSE_PREFILL = EnvBool(True)
|
||||
# gfx950 DeepSeek-V4.1 prefill: attend with aiter's OPUS sparse kernel over a bf16
|
||||
|
||||
@@ -443,6 +443,26 @@ def topk_transform_flashinfer_fused(
|
||||
)
|
||||
|
||||
|
||||
def topk_transform_paged_bf16(
|
||||
logits: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
page_table: torch.Tensor,
|
||||
out_page_indices: torch.Tensor,
|
||||
page_size: int,
|
||||
) -> None:
|
||||
"""DeepSelect paged top-k of bf16 logits: pool slots only, unordered."""
|
||||
from sglang.kernels.ops.attention.deep_select import topk_page_transform
|
||||
|
||||
topk_page_transform(
|
||||
logits,
|
||||
out_page_indices.shape[1],
|
||||
page_table=page_table,
|
||||
page_size=page_size,
|
||||
end=seq_lens.reshape(-1).to(torch.int32),
|
||||
output_idx=out_page_indices,
|
||||
)
|
||||
|
||||
|
||||
def topk_transform_paged_from_metadata(
|
||||
logits: torch.Tensor,
|
||||
metadata,
|
||||
@@ -464,6 +484,16 @@ def topk_transform_paged_from_metadata(
|
||||
page_table = metadata.page_table[rows]
|
||||
out_page_indices = page_indices[rows]
|
||||
out_raw_indices = raw_indices[rows] if raw_indices is not None else None
|
||||
if logits.dtype == torch.bfloat16:
|
||||
assert out_raw_indices is None
|
||||
topk_transform_paged_bf16(
|
||||
logits,
|
||||
seq_lens,
|
||||
page_table,
|
||||
out_page_indices,
|
||||
metadata.compressed_page_size,
|
||||
)
|
||||
return
|
||||
if metadata.use_topk_v2:
|
||||
topk_transform_paged_v2(
|
||||
logits,
|
||||
|
||||
@@ -14,6 +14,7 @@ from sglang.kernels.ops.attention.dsv4.candidate_blocks import (
|
||||
select_candidate_block_ids,
|
||||
topk_among_blocks,
|
||||
)
|
||||
from sglang.kernels.ops.attention.dsv4.index_logits import FLAT_LOGITS_WIDTH_ALIGN
|
||||
|
||||
from .scoring import (
|
||||
DeepGEMMPrefillData,
|
||||
@@ -23,6 +24,7 @@ from .scoring import (
|
||||
prefill_requests,
|
||||
score_tiles,
|
||||
select_decode,
|
||||
topk_ragged,
|
||||
write_decode,
|
||||
write_prefill,
|
||||
)
|
||||
@@ -246,14 +248,12 @@ def _publish_prefill_blocks(
|
||||
topk_blocks: int,
|
||||
block_size: int,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
from sglang.kernels.ops.attention.dsv4 import topk_transform_ragged_v2
|
||||
|
||||
selected = data.empty_selection(topk)
|
||||
width = min(topk_blocks, -(-max(data.lens_per_request, default=0) // block_size))
|
||||
blocks = torch.full(
|
||||
(data.num_rows, width), -1, dtype=torch.int32, device=selected.device
|
||||
)
|
||||
for tile, logits in score_tiles(data, kv, width_align=4):
|
||||
for tile, logits in score_tiles(data, kv):
|
||||
_publish_tile_blocks(
|
||||
data=data,
|
||||
tile=tile,
|
||||
@@ -262,11 +262,11 @@ def _publish_prefill_blocks(
|
||||
topk_blocks=topk_blocks,
|
||||
block_size=block_size,
|
||||
)
|
||||
topk_transform_ragged_v2(
|
||||
topk_ragged(
|
||||
logits,
|
||||
data.compress_lens[tile],
|
||||
out_offsets=data.request_starts[tile],
|
||||
out_indices=selected[tile],
|
||||
out=selected[tile],
|
||||
offsets=data.request_starts[tile],
|
||||
)
|
||||
# Free this tile's logits before the generator scores the next one.
|
||||
del logits
|
||||
@@ -307,7 +307,9 @@ def _consume_prefill_blocks(
|
||||
block_size: int,
|
||||
) -> torch.Tensor:
|
||||
selected = data.empty_selection(topk)
|
||||
for tile, logits in score_tiles(data, kv, width_align=block_size):
|
||||
# the block-aligned slices of `_consume_tile_blocks` stay within the logits width
|
||||
assert FLAT_LOGITS_WIDTH_ALIGN % block_size == 0
|
||||
for tile, logits in score_tiles(data, kv):
|
||||
_consume_tile_blocks(
|
||||
data=data,
|
||||
tile=tile,
|
||||
|
||||
@@ -5,7 +5,7 @@ a selection back into pool slots."""
|
||||
from __future__ import annotations
|
||||
|
||||
import itertools
|
||||
from typing import TYPE_CHECKING, Generator, Iterator, List, Optional, Tuple
|
||||
from typing import TYPE_CHECKING, Iterator, List, Optional, Tuple
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
@@ -207,12 +207,7 @@ def get_deep_gemm_prefill_data(
|
||||
)
|
||||
|
||||
|
||||
def score_tiles(
|
||||
data: DeepGEMMPrefillData,
|
||||
kv: Tuple[torch.Tensor, torch.Tensor],
|
||||
*,
|
||||
width_align: int,
|
||||
) -> Generator[Tuple[slice, torch.Tensor], None, None]:
|
||||
def score_tiles(data: DeepGEMMPrefillData, kv: Tuple[torch.Tensor, torch.Tensor]):
|
||||
yield from flat_index_logits_tiles(
|
||||
q=(data.q_fp4, data.q_sf),
|
||||
kv=kv,
|
||||
@@ -221,10 +216,35 @@ def score_tiles(
|
||||
lengths=data.compress_lens,
|
||||
context_lengths=data.lens_per_request,
|
||||
budget_bytes=_DEEP_GEMM_SCORE_BUDGET_BYTES,
|
||||
width_align=width_align,
|
||||
)
|
||||
|
||||
|
||||
def topk_ragged(
|
||||
logits: torch.Tensor,
|
||||
lens: torch.Tensor,
|
||||
*,
|
||||
out: torch.Tensor,
|
||||
offsets: torch.Tensor,
|
||||
) -> None:
|
||||
"""Top-k of each row's first ``lens`` columns plus ``offsets`` into ``out``,
|
||||
``-1`` padded, unordered: DeepSelect for bf16 logits, top-k v2 for fp32."""
|
||||
if logits.dtype == torch.bfloat16:
|
||||
from sglang.kernels.ops.attention.deep_select import topk
|
||||
|
||||
topk(
|
||||
logits,
|
||||
out.shape[1],
|
||||
end=lens,
|
||||
output_idx=out,
|
||||
output_idx_offset=offsets,
|
||||
idx_oob_fill_value=-1,
|
||||
)
|
||||
return
|
||||
from sglang.kernels.ops.attention.dsv4 import topk_transform_ragged_v2
|
||||
|
||||
topk_transform_ragged_v2(logits, lens, out_offsets=offsets, out_indices=out)
|
||||
|
||||
|
||||
def dense_prefill_topk(
|
||||
data: DeepGEMMPrefillData,
|
||||
kv: Tuple[torch.Tensor, torch.Tensor],
|
||||
@@ -232,16 +252,11 @@ def dense_prefill_topk(
|
||||
out: torch.Tensor,
|
||||
) -> None:
|
||||
"""Request-local compressed positions into ``out``, ``-1`` padded, unordered."""
|
||||
from sglang.kernels.ops.attention.dsv4 import topk_transform_ragged_v2
|
||||
|
||||
# Score columns are request-relative, so a zero offset emits request-local positions.
|
||||
zero_offsets = torch.zeros_like(data.request_starts)
|
||||
for tile, logits in score_tiles(data, kv, width_align=4):
|
||||
topk_transform_ragged_v2(
|
||||
logits,
|
||||
data.compress_lens[tile],
|
||||
out_offsets=zero_offsets[tile],
|
||||
out_indices=out[tile],
|
||||
for tile, logits in score_tiles(data, kv):
|
||||
topk_ragged(
|
||||
logits, data.compress_lens[tile], out=out[tile], offsets=zero_offsets[tile]
|
||||
)
|
||||
# Free this tile's logits before the generator scores the next one.
|
||||
del logits
|
||||
|
||||
@@ -28,7 +28,10 @@ from sglang.kernels.ops.attention.dsv4.topk import (
|
||||
topk_transform_ragged_v2,
|
||||
topk_transform_sparse,
|
||||
)
|
||||
from sglang.srt.layers.attention.dsv4.indexer import topk_transform_paged_from_metadata
|
||||
from sglang.srt.layers.attention.dsv4.indexer import (
|
||||
topk_transform_paged_bf16,
|
||||
topk_transform_paged_from_metadata,
|
||||
)
|
||||
from sglang.srt.layers.attention.dsv4.metadata import expand_index_page_table
|
||||
from sglang.srt.utils.common import async_h2d
|
||||
|
||||
@@ -39,6 +42,7 @@ from .scoring import (
|
||||
get_deep_gemm_prefill_data,
|
||||
get_index_k_cache,
|
||||
score_tiles,
|
||||
topk_ragged,
|
||||
)
|
||||
from .types import (
|
||||
CandidateMetadata,
|
||||
@@ -188,14 +192,23 @@ class SparseTableBackend:
|
||||
# The block-selection chain reads logits after the main stream moves on.
|
||||
logits.record_stream(self.alt_stream)
|
||||
# TODO(candidate): one kernel for both selections below (dense logits read once)
|
||||
topk_transform_paged_v2(
|
||||
logits,
|
||||
seq_lens,
|
||||
metadata.page_table,
|
||||
inputs.out_page_indices,
|
||||
metadata.compressed_page_size,
|
||||
metadata.topk_metadata,
|
||||
)
|
||||
if logits.dtype == torch.bfloat16:
|
||||
topk_transform_paged_bf16(
|
||||
logits,
|
||||
seq_lens,
|
||||
metadata.page_table,
|
||||
inputs.out_page_indices,
|
||||
metadata.compressed_page_size,
|
||||
)
|
||||
else:
|
||||
topk_transform_paged_v2(
|
||||
logits,
|
||||
seq_lens,
|
||||
metadata.page_table,
|
||||
inputs.out_page_indices,
|
||||
metadata.compressed_page_size,
|
||||
metadata.topk_metadata,
|
||||
)
|
||||
# per row: block count for the block top-k, sparse-row length for the consumers
|
||||
with torch.cuda.stream(self.alt_stream):
|
||||
nblocks, row_valid_lens = candidate_row_lens(seq_lens, self.topk_blocks)
|
||||
@@ -403,20 +416,17 @@ def publish_prefill_table(
|
||||
blocks = torch.empty(rows, topk_blocks, dtype=torch.int32, device=device)
|
||||
zero_offsets = torch.zeros(rows, dtype=torch.int32, device=device)
|
||||
# the block keys read the score rows through 32-byte vectors
|
||||
for tile, logits in score_tiles(data, kv, width_align=8):
|
||||
for tile, logits in score_tiles(data, kv):
|
||||
lens = data.compress_lens[tile]
|
||||
topk_transform_ragged_v2(
|
||||
logits,
|
||||
lens,
|
||||
out_offsets=zero_offsets[tile],
|
||||
out_indices=out_positions[tile],
|
||||
)
|
||||
topk_ragged(logits, lens, out=out_positions[tile], offsets=zero_offsets[tile])
|
||||
# keys past a row's block count stay unset; the top-k reads a row
|
||||
# up to its block count only (v2 wants the stride a multiple of 4)
|
||||
width_blocks = (
|
||||
logits.shape[1] + CANDIDATE_BLOCK_SIZE - 1
|
||||
) // CANDIDATE_BLOCK_SIZE
|
||||
keys = logits.new_empty(logits.shape[0], (width_blocks + 3) // 4 * 4)
|
||||
keys = logits.new_empty(
|
||||
logits.shape[0], (width_blocks + 3) // 4 * 4, dtype=torch.float32
|
||||
)
|
||||
amax8_varlen(logits, lens, out=keys)
|
||||
topk_transform_ragged_v2(
|
||||
keys,
|
||||
|
||||
Reference in New Issue
Block a user