dsv4.1-amd: gfx950 sparse decode attention and sorted top-k (#41020)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
26d7539799
commit
096b066fb4
@@ -0,0 +1,76 @@
|
||||
"""Benchmark for the DeepSeek-V4 indexer top-k transform, with and without sorted output."""
|
||||
|
||||
import itertools
|
||||
|
||||
import torch
|
||||
import triton
|
||||
import triton.testing
|
||||
from sgl_kernel import deepseek_v4_topk_transform_512
|
||||
|
||||
try:
|
||||
from sglang.utils import is_in_ci
|
||||
|
||||
IS_CI = is_in_ci()
|
||||
except ImportError:
|
||||
IS_CI = False
|
||||
|
||||
TOPK = 512
|
||||
PAGE_SIZE = 64
|
||||
|
||||
batch_sizes = [1] if IS_CI else [1, 8, 32, 128]
|
||||
seq_lens = [4096] if IS_CI else [1024, 4096, 32768]
|
||||
|
||||
configs = list(itertools.product(batch_sizes, seq_lens))
|
||||
|
||||
|
||||
@triton.testing.perf_report(
|
||||
triton.testing.Benchmark(
|
||||
x_names=["batch_size", "seq_len"],
|
||||
x_vals=configs,
|
||||
line_arg="provider",
|
||||
line_vals=["unsorted", "sorted", "torch_sort"],
|
||||
line_names=["SGL Kernel", "SGL Kernel sort_output", "SGL Kernel + torch.sort"],
|
||||
styles=[("blue", "-"), ("green", "-"), ("red", "--")],
|
||||
ylabel="µs (median)",
|
||||
plot_name="dsv4-topk-transform-performance",
|
||||
args={},
|
||||
)
|
||||
)
|
||||
def benchmark_topk_transform(batch_size, seq_len, provider):
|
||||
torch.manual_seed(42)
|
||||
scores = torch.randn(batch_size, seq_len, dtype=torch.float32, device="cuda")
|
||||
lengths = torch.full((batch_size,), seq_len, dtype=torch.int32, device="cuda")
|
||||
num_pages = triton.cdiv(seq_len, PAGE_SIZE)
|
||||
page_table = torch.stack(
|
||||
[
|
||||
torch.randperm(4 * num_pages, device="cuda")[:num_pages]
|
||||
for _ in range(batch_size)
|
||||
]
|
||||
).int()
|
||||
page_indices = torch.empty(batch_size, TOPK, dtype=torch.int32, device="cuda")
|
||||
|
||||
def topk(sort_output):
|
||||
deepseek_v4_topk_transform_512(
|
||||
scores,
|
||||
lengths,
|
||||
page_table,
|
||||
page_indices,
|
||||
PAGE_SIZE,
|
||||
sort_output=sort_output,
|
||||
)
|
||||
|
||||
if provider == "unsorted":
|
||||
fn = lambda: topk(False)
|
||||
elif provider == "sorted":
|
||||
fn = lambda: topk(True)
|
||||
else:
|
||||
fn = lambda: (topk(False), torch.sort(page_indices, dim=-1))
|
||||
|
||||
ms, min_ms, max_ms = triton.testing.do_bench_cudagraph(
|
||||
fn, quantiles=[0.5, 0.2, 0.8]
|
||||
)
|
||||
return 1000 * ms, 1000 * max_ms, 1000 * min_ms
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
benchmark_topk_transform.run(print_data=True)
|
||||
@@ -49,7 +49,7 @@ TORCH_LIBRARY_EXPAND(sgl_kernel, m) {
|
||||
|
||||
m.def(
|
||||
"deepseek_v4_topk_transform_512(Tensor scores, Tensor seq_lens, Tensor page_table, Tensor! "
|
||||
"page_indices, int page_size, Tensor!? raw_indices) -> ()");
|
||||
"page_indices, int page_size, Tensor!? raw_indices, bool sort_output=False) -> ()");
|
||||
m.impl("deepseek_v4_topk_transform_512", torch::kCUDA, &deepseek_v4_topk_transform_512);
|
||||
|
||||
m.def(
|
||||
|
||||
@@ -46,6 +46,7 @@ constexpr size_t kSMEM = 48 * 1024; // bytes
|
||||
#endif
|
||||
static_assert(kSMEM % (2 * sizeof(int32_t)) == 0, "kSMEM must be a multiple of 8 bytes.");
|
||||
|
||||
// seq_lens[b] must not exceed scores.size(1) or page_table.size(1) << page_bits: a row reads up to its length
|
||||
struct TopKParams {
|
||||
const float* __restrict__ scores;
|
||||
const int32_t* __restrict__ seq_lens;
|
||||
@@ -57,6 +58,8 @@ struct TopKParams {
|
||||
uint32_t page_bits;
|
||||
uint32_t topk;
|
||||
int64_t output_stride;
|
||||
// Emit each row's picks in ascending order (see bitonic_sort_u32).
|
||||
bool sort_output;
|
||||
};
|
||||
|
||||
__device__ __forceinline__ uint8_t convert_to_uint8(float x) {
|
||||
@@ -251,6 +254,116 @@ radix_topk(const float* __restrict__ input, int32_t* __restrict__ output, uint32
|
||||
}
|
||||
}
|
||||
|
||||
#ifdef USE_ROCM
|
||||
// Bitonic sort of n (a power of two, 64 <= n <= kMaxTopK) 32-bit keys, one per thread: strides
|
||||
// below the wavefront width exchange through lane shuffles, the wider ones through LDS.
|
||||
#if defined(__GFX10__) || defined(__GFX11__) || defined(__GFX12__)
|
||||
constexpr uint32_t kWaveSize = 32; // RDNA and gfx1250 run 32-lane waves
|
||||
#else
|
||||
constexpr uint32_t kWaveSize = 64;
|
||||
#endif
|
||||
|
||||
// lane ^ J's value in registers: DPP for J <= 8, gfx950 permlane swaps for J = 16, 32; __shfl_xor
|
||||
// is an LDS round trip on every stage's dependent chain.
|
||||
template <uint32_t J>
|
||||
__device__ __forceinline__ uint32_t lane_xor(uint32_t v) {
|
||||
#if defined(__gfx90a__) || defined(__gfx942__) || defined(__gfx950__)
|
||||
if constexpr (J == 1) { // quad_perm [1, 0, 3, 2]
|
||||
return static_cast<uint32_t>(__builtin_amdgcn_update_dpp(0, static_cast<int>(v), 0xB1, 0xF, 0xF, true));
|
||||
}
|
||||
if constexpr (J == 2) { // quad_perm [2, 3, 0, 1]
|
||||
return static_cast<uint32_t>(__builtin_amdgcn_update_dpp(0, static_cast<int>(v), 0x4E, 0xF, 0xF, true));
|
||||
}
|
||||
if constexpr (J == 4 || J == 8) {
|
||||
// within a 16-lane row: bit clear reads J lanes up (row_shl), bit set J lanes down (row_shr)
|
||||
const uint32_t shl =
|
||||
static_cast<uint32_t>(__builtin_amdgcn_update_dpp(0, static_cast<int>(v), 0x100 | J, 0xF, 0xF, true));
|
||||
const uint32_t shr =
|
||||
static_cast<uint32_t>(__builtin_amdgcn_update_dpp(0, static_cast<int>(v), 0x110 | J, 0xF, 0xF, true));
|
||||
return (__lane_id() & J) ? shr : shl;
|
||||
}
|
||||
#endif
|
||||
#if defined(__gfx950__)
|
||||
if constexpr (J == 16) {
|
||||
// rows 1 and 3 of the first operand swap with rows 0 and 2 of the second
|
||||
const auto pair = __builtin_amdgcn_permlane16_swap(v, v, false, false);
|
||||
return (__lane_id() & 16) ? pair[0] : pair[1];
|
||||
}
|
||||
if constexpr (J == 32) {
|
||||
const auto pair = __builtin_amdgcn_permlane32_swap(v, v, false, false);
|
||||
return (__lane_id() & 32) ? pair[0] : pair[1];
|
||||
}
|
||||
#endif
|
||||
return static_cast<uint32_t>(__shfl_xor(static_cast<int>(v), static_cast<int>(J), kWaveSize));
|
||||
}
|
||||
|
||||
// One bitonic stage (merge size K, stride J); templated so the network unrolls with no stride switch.
|
||||
template <uint32_t K, uint32_t J>
|
||||
__device__ __forceinline__ uint32_t bitonic_stage(uint32_t v, uint32_t* __restrict__ s_vals, uint32_t tx, uint32_t n) {
|
||||
const bool up = (tx & K) == 0;
|
||||
const bool lower = (tx & J) == 0;
|
||||
uint32_t w;
|
||||
if constexpr (J >= kWaveSize) {
|
||||
__syncthreads();
|
||||
if (tx < n) s_vals[tx] = v;
|
||||
__syncthreads();
|
||||
w = tx < n ? s_vals[tx ^ J] : ~0u;
|
||||
} else {
|
||||
w = lane_xor<J>(v);
|
||||
}
|
||||
return (lower == up) ? min(v, w) : max(v, w);
|
||||
}
|
||||
|
||||
template <uint32_t N, uint32_t K, uint32_t J>
|
||||
__device__ __forceinline__ uint32_t bitonic_network(uint32_t v, uint32_t* __restrict__ s_vals, uint32_t tx) {
|
||||
v = bitonic_stage<K, J>(v, s_vals, tx, N);
|
||||
if constexpr (J > 1) {
|
||||
return bitonic_network<N, K, J / 2>(v, s_vals, tx);
|
||||
} else if constexpr (K < N) {
|
||||
return bitonic_network<N, K * 2, K>(v, s_vals, tx);
|
||||
} else {
|
||||
return v;
|
||||
}
|
||||
}
|
||||
|
||||
// Sort the N (a power of two, 64 <= N <= kBlockSize) values in s_vals ascending, one per thread.
|
||||
template <uint32_t N>
|
||||
__device__ void bitonic_sort_fixed(uint32_t* __restrict__ s_vals) {
|
||||
static_assert(N >= 64 && N <= kBlockSize && (N & (N - 1)) == 0, "one value per thread");
|
||||
const uint32_t tx = threadIdx.x;
|
||||
uint32_t v = tx < N ? s_vals[tx] : ~0u;
|
||||
v = bitonic_network<N, 2, 1>(v, s_vals, tx);
|
||||
__syncthreads();
|
||||
if (tx < N) s_vals[tx] = v;
|
||||
__syncthreads();
|
||||
}
|
||||
|
||||
// one value per thread: the block covers the largest top-k
|
||||
static_assert(kBlockSize >= kMaxTopK);
|
||||
|
||||
__device__ void bitonic_sort_u32(uint32_t* __restrict__ s_vals, uint32_t n) {
|
||||
switch (n) {
|
||||
case 64:
|
||||
return bitonic_sort_fixed<64>(s_vals);
|
||||
case 128:
|
||||
return bitonic_sort_fixed<128>(s_vals);
|
||||
case 256:
|
||||
return bitonic_sort_fixed<256>(s_vals);
|
||||
case 512:
|
||||
return bitonic_sort_fixed<512>(s_vals);
|
||||
default:
|
||||
return bitonic_sort_fixed<kMaxTopK>(s_vals);
|
||||
}
|
||||
}
|
||||
|
||||
__device__ __forceinline__ uint32_t next_pow2_at_least_64(uint32_t x) {
|
||||
uint32_t n = 64;
|
||||
while (n < x)
|
||||
n <<= 1;
|
||||
return n;
|
||||
}
|
||||
#endif // USE_ROCM
|
||||
|
||||
__global__ __launch_bounds__(kBlockSize) void deepseek_v4_topk_transform_kernel(const TopKParams params) {
|
||||
const auto bid = blockIdx.x;
|
||||
const auto seq_len = params.seq_lens[bid];
|
||||
@@ -261,15 +374,63 @@ __global__ __launch_bounds__(kBlockSize) void deepseek_v4_topk_transform_kernel(
|
||||
const auto raw_indices_ptr =
|
||||
params.raw_indices != nullptr ? params.raw_indices + bid * params.output_stride : nullptr;
|
||||
|
||||
__shared__ int32_t s_topk_indices[kMaxTopK];
|
||||
|
||||
// key: the position when the row has raw indices, else the slot
|
||||
const bool key_is_position = raw_indices_ptr != nullptr;
|
||||
uint32_t count = topk;
|
||||
if (seq_len <= static_cast<int32_t>(topk)) {
|
||||
naive_paged_transform(seq_len, topk, params.page_bits, page_ptr, indices_ptr, raw_indices_ptr);
|
||||
if (!params.sort_output || key_is_position) {
|
||||
// ascending positions with the -1 padding last: already the sorted row
|
||||
naive_paged_transform(seq_len, topk, params.page_bits, page_ptr, indices_ptr, raw_indices_ptr);
|
||||
return;
|
||||
}
|
||||
// every position is a pick; the slot order still has to be established
|
||||
count = static_cast<uint32_t>(seq_len);
|
||||
for (uint32_t i = threadIdx.x; i < count; i += kBlockSize) {
|
||||
s_topk_indices[i] = static_cast<int32_t>(i);
|
||||
}
|
||||
} else {
|
||||
radix_topk(score_ptr, s_topk_indices, static_cast<uint32_t>(seq_len), topk);
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
#ifdef USE_ROCM
|
||||
if (params.sort_output) {
|
||||
__shared__ uint32_t s_sort_vals[kMaxTopK];
|
||||
const uint32_t n = next_pow2_at_least_64(count);
|
||||
for (uint32_t i = threadIdx.x; i < n; i += kBlockSize) {
|
||||
uint32_t key = ~0u;
|
||||
if (i < count) {
|
||||
const int32_t raw = s_topk_indices[i];
|
||||
key = static_cast<uint32_t>(
|
||||
key_is_position ? raw : page_to_slot(page_ptr, static_cast<uint32_t>(raw), params.page_bits));
|
||||
}
|
||||
s_sort_vals[i] = key;
|
||||
}
|
||||
__syncthreads();
|
||||
bitonic_sort_u32(s_sort_vals, n);
|
||||
for (uint32_t i = threadIdx.x; i < topk; i += kBlockSize) {
|
||||
int32_t slot = -1;
|
||||
int32_t raw = -1;
|
||||
if (i < count) {
|
||||
const int32_t key = static_cast<int32_t>(s_sort_vals[i]);
|
||||
if (key_is_position) {
|
||||
raw = key;
|
||||
slot = page_to_slot(page_ptr, static_cast<uint32_t>(raw), params.page_bits);
|
||||
} else {
|
||||
slot = key;
|
||||
}
|
||||
}
|
||||
indices_ptr[i] = slot;
|
||||
if (raw_indices_ptr != nullptr) {
|
||||
raw_indices_ptr[i] = raw;
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
#endif
|
||||
|
||||
__shared__ int32_t s_topk_indices[kMaxTopK];
|
||||
radix_topk(score_ptr, s_topk_indices, static_cast<uint32_t>(seq_len), topk);
|
||||
|
||||
__syncthreads();
|
||||
for (uint32_t i = threadIdx.x; i < topk; i += kBlockSize) {
|
||||
const auto raw = s_topk_indices[i];
|
||||
indices_ptr[i] = page_to_slot(page_ptr, static_cast<uint32_t>(raw), params.page_bits);
|
||||
@@ -304,11 +465,15 @@ void deepseek_v4_topk_transform_512(
|
||||
const at::Tensor& page_table,
|
||||
at::Tensor& page_indices,
|
||||
int64_t page_size,
|
||||
std::optional<at::Tensor> raw_indices_opt) {
|
||||
std::optional<at::Tensor> raw_indices_opt,
|
||||
bool sort_output) {
|
||||
CHECK_CUDA(scores);
|
||||
CHECK_CUDA(seq_lens);
|
||||
CHECK_CUDA(page_table);
|
||||
CHECK_CUDA(page_indices);
|
||||
#ifndef USE_ROCM
|
||||
TORCH_CHECK(!sort_output, "deepseek_v4_topk_transform_512: sort_output is only built for ROCm");
|
||||
#endif
|
||||
if (raw_indices_opt.has_value()) {
|
||||
CHECK_CUDA(raw_indices_opt.value());
|
||||
}
|
||||
@@ -367,6 +532,7 @@ void deepseek_v4_topk_transform_512(
|
||||
.page_bits = page_bits,
|
||||
.topk = static_cast<uint32_t>(topk),
|
||||
.output_stride = topk,
|
||||
.sort_output = sort_output,
|
||||
};
|
||||
|
||||
const auto stream = at::cuda::getCurrentCUDAStream().stream();
|
||||
|
||||
@@ -170,7 +170,8 @@ void deepseek_v4_topk_transform_512(
|
||||
const at::Tensor& page_table,
|
||||
at::Tensor& page_indices,
|
||||
int64_t page_size,
|
||||
std::optional<at::Tensor> raw_indices_opt = std::nullopt);
|
||||
std::optional<at::Tensor> raw_indices_opt = std::nullopt,
|
||||
bool sort_output = false);
|
||||
#endif
|
||||
|
||||
/*
|
||||
|
||||
@@ -87,6 +87,7 @@ def deepseek_v4_topk_transform_512(
|
||||
page_indices: torch.Tensor,
|
||||
page_size: int,
|
||||
raw_indices: Optional[torch.Tensor] = None,
|
||||
sort_output: bool = False,
|
||||
) -> None:
|
||||
"""
|
||||
Performs the DeepSeek-V4 indexer top-k selection and writes the paged
|
||||
@@ -96,7 +97,8 @@ def deepseek_v4_topk_transform_512(
|
||||
|
||||
Args:
|
||||
scores: float32 ``[B, max_seq_len]`` indexer logits, contiguous on dim 1.
|
||||
seq_lens: int32 ``[B]``, true KV length per batch row.
|
||||
seq_lens: int32 ``[B]``, true KV length per batch row; each at most
|
||||
``max_seq_len`` and ``num_pages * page_size``.
|
||||
page_table: int32 ``[B, num_pages]``, logical->physical page table,
|
||||
contiguous on dim 1.
|
||||
page_indices: int32 ``[B, topk]``, output buffer, contiguous. Filled
|
||||
@@ -104,11 +106,14 @@ def deepseek_v4_topk_transform_512(
|
||||
page_size: power-of-2 page size.
|
||||
raw_indices: optional int32 ``[B, topk]``, contiguous. If provided,
|
||||
filled with raw token positions within each row.
|
||||
sort_output: sort every row ascending in the kernel epilogue, with the
|
||||
-1 padding last: by raw position when ``raw_indices`` is given,
|
||||
otherwise by physical slot. The selected set is unchanged.
|
||||
"""
|
||||
if raw_indices is not None:
|
||||
assert raw_indices.dim() == 2
|
||||
torch.ops.sgl_kernel.deepseek_v4_topk_transform_512(
|
||||
scores, seq_lens, page_table, page_indices, page_size, raw_indices
|
||||
scores, seq_lens, page_table, page_indices, page_size, raw_indices, sort_output
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -292,6 +292,51 @@ def test_deepseek_v4_topk_transform(bs: int, c4_len: int) -> None:
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
torch.version.hip is None,
|
||||
reason="deepseek_v4_topk_transform_512 is only built on ROCm",
|
||||
)
|
||||
@pytest.mark.parametrize("with_raw", [False, True])
|
||||
@torch.inference_mode()
|
||||
def test_deepseek_v4_topk_transform_sorted_output(with_raw: bool) -> None:
|
||||
"""sort_output orders each row ascending (by raw position when given), -1 padding last,
|
||||
without changing the selected set, also for rows shorter than topk."""
|
||||
from sgl_kernel import deepseek_v4_topk_transform_512
|
||||
|
||||
torch.manual_seed(7)
|
||||
topk, page_size, width = 512, 64, 5000
|
||||
lens = [0, 1, topk - 1, topk, topk + 1, 3000, width]
|
||||
bs = len(lens)
|
||||
scores = torch.randn(bs, width, dtype=torch.float32, device="cuda")
|
||||
seq_lens = torch.tensor(lens, dtype=torch.int32, device="cuda")
|
||||
num_pages = (width + page_size - 1) // page_size
|
||||
page_table = torch.stack(
|
||||
[torch.randperm(4 * num_pages, device="cuda")[:num_pages] for _ in range(bs)]
|
||||
).int()
|
||||
|
||||
outs = []
|
||||
for sort_output in (False, True):
|
||||
page = torch.empty(bs, topk, dtype=torch.int32, device="cuda")
|
||||
raw = torch.empty_like(page) if with_raw else None
|
||||
deepseek_v4_topk_transform_512(
|
||||
scores, seq_lens, page_table, page, page_size, raw, sort_output=sort_output
|
||||
)
|
||||
outs.append((page, raw))
|
||||
(page, raw), (page_sorted, raw_sorted) = outs
|
||||
for b in range(bs):
|
||||
key = (raw if with_raw else page)[b]
|
||||
chosen = key[key >= 0].sort().values
|
||||
padding = torch.full(
|
||||
(topk - chosen.numel(),), -1, dtype=torch.int32, device="cuda"
|
||||
)
|
||||
assert torch.equal(
|
||||
(raw_sorted if with_raw else page_sorted)[b], torch.cat([chosen, padding])
|
||||
)
|
||||
if with_raw: # each slot still travels with its raw position
|
||||
pairs = sorted(zip(page[b].tolist(), raw[b].tolist()))
|
||||
assert pairs == sorted(zip(page_sorted[b].tolist(), raw_sorted[b].tolist()))
|
||||
|
||||
|
||||
def _make_scores(kind: str, bs: int, width: int, seed: int) -> torch.Tensor:
|
||||
"""Score distributions that stress the coarse stage of a histogram top-k.
|
||||
|
||||
|
||||
@@ -0,0 +1,426 @@
|
||||
"""Single-launch replacements for the torch glue around the DeepSeek-V4.1 sparse attention metadata
|
||||
on HIP; every kernel reproduces the torch expression quoted in its docstring bit for bit."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Dict, Optional, Tuple
|
||||
|
||||
import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _mask_indices_by_length_kernel(
|
||||
idx_ptr,
|
||||
len_ptr,
|
||||
out_ptr,
|
||||
idx2_ptr,
|
||||
len2_ptr,
|
||||
out2_ptr,
|
||||
rows_per_len,
|
||||
W1: tl.constexpr,
|
||||
W2: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
HAS_SECOND: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0)
|
||||
which = tl.program_id(1)
|
||||
length_row = row // rows_per_len
|
||||
offs = tl.arange(0, BLOCK)
|
||||
if which == 0:
|
||||
length = tl.load(len_ptr + length_row)
|
||||
for w0 in tl.static_range(0, W1, BLOCK):
|
||||
w = w0 + offs
|
||||
m = w < W1
|
||||
v = tl.load(idx_ptr + row * W1 + w, mask=m, other=0)
|
||||
tl.store(out_ptr + row * W1 + w, tl.where(w < length, v, -1), mask=m)
|
||||
elif HAS_SECOND:
|
||||
length = tl.load(len2_ptr + length_row)
|
||||
for w0 in tl.static_range(0, W2, BLOCK):
|
||||
w = w0 + offs
|
||||
m = w < W2
|
||||
v = tl.load(idx2_ptr + row * W2 + w, mask=m, other=0)
|
||||
tl.store(out2_ptr + row * W2 + w, tl.where(w < length, v, -1), mask=m)
|
||||
|
||||
|
||||
def _mask_shape(indices: torch.Tensor, lengths: torch.Tensor) -> Tuple[int, int, int]:
|
||||
assert indices.is_contiguous(), indices.stride()
|
||||
assert lengths.dim() == 1 and lengths.is_contiguous()
|
||||
w = indices.shape[-1]
|
||||
rows = indices.numel() // w if w else 0
|
||||
batch = lengths.shape[0]
|
||||
assert batch > 0 and rows % batch == 0, (indices.shape, lengths.shape)
|
||||
return rows, rows // batch, w
|
||||
|
||||
|
||||
def mask_indices_by_length(
|
||||
indices: torch.Tensor,
|
||||
lengths: torch.Tensor,
|
||||
indices2: Optional[torch.Tensor] = None,
|
||||
lengths2: Optional[torch.Tensor] = None,
|
||||
) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
|
||||
"""torch.where(arange(w) < lengths.view(-1, 1, 1), indices, -1) for one or
|
||||
two [b, s, w] index tensors in one launch (the sparse decode kernel takes
|
||||
no per-row length and skips -1). lengths is [b]; every s row of a
|
||||
batch entry shares its length. Returns new tensors of the input dtypes."""
|
||||
rows, rows_per_len, w1 = _mask_shape(indices, lengths)
|
||||
out = torch.empty_like(indices)
|
||||
has_second = indices2 is not None
|
||||
if has_second:
|
||||
rows2, rows_per_len2, w2 = _mask_shape(indices2, lengths2)
|
||||
assert rows2 == rows and rows_per_len2 == rows_per_len, (
|
||||
indices.shape,
|
||||
indices2.shape,
|
||||
)
|
||||
out2 = torch.empty_like(indices2)
|
||||
else:
|
||||
w2 = w1
|
||||
indices2, lengths2, out2 = indices, lengths, out
|
||||
if rows == 0:
|
||||
return out, (out2 if has_second else None)
|
||||
block = min(1024, triton.next_power_of_2(max(w1, w2)))
|
||||
_mask_indices_by_length_kernel[(rows, 2 if has_second else 1)](
|
||||
indices,
|
||||
lengths,
|
||||
out,
|
||||
indices2,
|
||||
lengths2,
|
||||
out2,
|
||||
rows_per_len,
|
||||
W1=w1,
|
||||
W2=w2,
|
||||
BLOCK=block,
|
||||
HAS_SECOND=has_second,
|
||||
num_warps=4,
|
||||
)
|
||||
return out, (out2 if has_second else None)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _expand_index_page_table_kernel(
|
||||
pt_ptr,
|
||||
out_ptr,
|
||||
n_pages,
|
||||
pt_stride,
|
||||
BPP: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0)
|
||||
p = tl.program_id(1) * BLOCK + tl.arange(0, BLOCK)
|
||||
m = p < n_pages
|
||||
page = tl.load(pt_ptr + row * pt_stride + p, mask=m, other=0)
|
||||
base = page.to(tl.int64) * BPP
|
||||
k = tl.arange(0, BPP)
|
||||
expanded = base[:, None] + k[None, :]
|
||||
tl.store(
|
||||
out_ptr + row * (n_pages * BPP) + p[:, None] * BPP + k[None, :],
|
||||
expanded.to(tl.int32),
|
||||
mask=m[:, None],
|
||||
)
|
||||
|
||||
|
||||
def expand_index_page_table(
|
||||
page_table: torch.Tensor, blocks_per_page: int
|
||||
) -> torch.Tensor:
|
||||
"""(page_table.to(int64) * bpp).unsqueeze(-1) + arange(bpp) reshaped to
|
||||
[bs, n * bpp] int32: the block table of a low-ratio indexer-K pool that
|
||||
pages at a fraction of a FULL page (candidate_indexer.expand_index_page_table)."""
|
||||
if blocks_per_page == 1:
|
||||
return page_table
|
||||
assert blocks_per_page & (blocks_per_page - 1) == 0, blocks_per_page
|
||||
assert page_table.dim() == 2 and page_table.stride(1) == 1
|
||||
bs, n = page_table.shape
|
||||
out = torch.empty(
|
||||
(bs, n * blocks_per_page), dtype=torch.int32, device=page_table.device
|
||||
)
|
||||
if bs == 0 or n == 0:
|
||||
return out
|
||||
block = 256
|
||||
_expand_index_page_table_kernel[(bs, triton.cdiv(n, block))](
|
||||
page_table,
|
||||
out,
|
||||
n,
|
||||
page_table.stride(0),
|
||||
BPP=blocks_per_page,
|
||||
BLOCK=block,
|
||||
num_warps=4,
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _low_ratio_compression_metadata_kernel(
|
||||
seq_lens_ptr,
|
||||
raw_out_loc_ptr,
|
||||
c1_out_loc_ptr,
|
||||
c1_clamp1_ptr,
|
||||
c2_out_loc_ptr,
|
||||
c2_clamp1_ptr,
|
||||
rows,
|
||||
num_write_tokens,
|
||||
HAS_C1: tl.constexpr,
|
||||
HAS_C2: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
r = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)
|
||||
m = r < rows
|
||||
mw = r < num_write_tokens
|
||||
seq_len = tl.load(seq_lens_ptr + r, mask=m, other=0)
|
||||
raw_out_loc = tl.load(raw_out_loc_ptr + r, mask=mw, other=0).to(tl.int64)
|
||||
if HAS_C1:
|
||||
# ratio 1: every token completes a group, so out_loc is raw_out_loc itself
|
||||
tl.store(c1_out_loc_ptr + r, raw_out_loc, mask=mw)
|
||||
tl.store(c1_clamp1_ptr + r, tl.maximum(seq_len, 1), mask=m)
|
||||
if HAS_C2:
|
||||
completes = (seq_len % 2) == 0
|
||||
# >> 1 floors as torch's // does; Triton's // rounds toward zero on a negative loc
|
||||
tl.store(c2_out_loc_ptr + r, tl.where(completes, raw_out_loc >> 1, -1), mask=mw)
|
||||
tl.store(c2_clamp1_ptr + r, tl.maximum(seq_len // 2, 1), mask=m)
|
||||
|
||||
|
||||
def low_ratio_compression_metadata(
|
||||
seq_lens_casual: torch.Tensor,
|
||||
raw_out_loc: torch.Tensor,
|
||||
low_ratios: Tuple[int, ...],
|
||||
) -> Dict[str, torch.Tensor]:
|
||||
"""_low_ratio_compression_metadata for ratios 1 and 2 in one launch: cR_out_loc =
|
||||
where(seq_lens[:nw] % R == 0, raw_out_loc // R, -1) (int64) and cR_topk_lengths_clamp1 =
|
||||
(seq_lens // R).clamp_min(1) (int32), keyed c{R}_... for the ratios present."""
|
||||
assert seq_lens_casual.dtype is torch.int32 and seq_lens_casual.is_contiguous()
|
||||
assert raw_out_loc.dim() == 1 and raw_out_loc.is_contiguous()
|
||||
assert set(low_ratios) <= {1, 2}, low_ratios
|
||||
rows = seq_lens_casual.shape[0]
|
||||
nw = raw_out_loc.shape[0]
|
||||
assert nw <= rows, (nw, rows)
|
||||
device = seq_lens_casual.device
|
||||
out = {}
|
||||
for ratio in (1, 2):
|
||||
if ratio in low_ratios:
|
||||
out[f"c{ratio}_out_loc"] = torch.empty(nw, dtype=torch.int64, device=device)
|
||||
out[f"c{ratio}_topk_lengths_clamp1"] = torch.empty(
|
||||
rows, dtype=torch.int32, device=device
|
||||
)
|
||||
if not out or rows == 0:
|
||||
return out
|
||||
placeholder = seq_lens_casual
|
||||
_low_ratio_compression_metadata_kernel[(triton.cdiv(rows, 1024),)](
|
||||
seq_lens_casual,
|
||||
raw_out_loc,
|
||||
out.get("c1_out_loc", placeholder),
|
||||
out.get("c1_topk_lengths_clamp1", placeholder),
|
||||
out.get("c2_out_loc", placeholder),
|
||||
out.get("c2_topk_lengths_clamp1", placeholder),
|
||||
rows,
|
||||
nw,
|
||||
HAS_C1=1 in low_ratios,
|
||||
HAS_C2=2 in low_ratios,
|
||||
BLOCK=1024,
|
||||
num_warps=4,
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _sparse_buffers_kernel(
|
||||
c4_clamp1_ptr,
|
||||
c4_raw_ptr,
|
||||
c4_sparse_ptr,
|
||||
c4_sparse_raw_ptr,
|
||||
c4_page_ptr,
|
||||
c1_clamp1_ptr,
|
||||
c1_sparse_ptr,
|
||||
c1_page_ptr,
|
||||
c2_clamp1_ptr,
|
||||
c2_sparse_ptr,
|
||||
c2_page_ptr,
|
||||
rows,
|
||||
topk,
|
||||
width,
|
||||
HAS_C1: tl.constexpr,
|
||||
HAS_C2: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
FILL_BLOCK: tl.constexpr,
|
||||
):
|
||||
pid = tl.program_id(0)
|
||||
if pid == 0:
|
||||
# one program clamps the per-row lengths: a few ints per row
|
||||
for r0 in range(0, rows, BLOCK):
|
||||
r = r0 + tl.arange(0, BLOCK)
|
||||
m = r < rows
|
||||
c4_clamp1 = tl.load(c4_clamp1_ptr + r, mask=m, other=0)
|
||||
c4_raw = tl.load(c4_raw_ptr + r, mask=m, other=0)
|
||||
tl.store(c4_sparse_ptr + r, tl.minimum(c4_clamp1, topk), mask=m)
|
||||
tl.store(c4_sparse_raw_ptr + r, tl.minimum(c4_raw, topk), mask=m)
|
||||
if HAS_C1:
|
||||
c1_clamp1 = tl.load(c1_clamp1_ptr + r, mask=m, other=0)
|
||||
tl.store(c1_sparse_ptr + r, tl.minimum(c1_clamp1, topk), mask=m)
|
||||
if HAS_C2:
|
||||
c2_clamp1 = tl.load(c2_clamp1_ptr + r, mask=m, other=0)
|
||||
tl.store(c2_sparse_ptr + r, tl.minimum(c2_clamp1, topk), mask=m)
|
||||
else:
|
||||
# the rest fill the -1 page-index buffers, one buffer per range of programs
|
||||
fill = pid - 1
|
||||
n = rows * width
|
||||
per_buffer = tl.cdiv(n, FILL_BLOCK)
|
||||
which = fill // per_buffer
|
||||
i = (fill - which * per_buffer) * FILL_BLOCK + tl.arange(0, FILL_BLOCK)
|
||||
m = i < n
|
||||
if which == 0:
|
||||
tl.store(c4_page_ptr + i, -1, mask=m)
|
||||
elif which == 1:
|
||||
if HAS_C1:
|
||||
tl.store(c1_page_ptr + i, -1, mask=m)
|
||||
elif HAS_C2:
|
||||
tl.store(c2_page_ptr + i, -1, mask=m)
|
||||
else:
|
||||
if HAS_C1 and HAS_C2:
|
||||
tl.store(c2_page_ptr + i, -1, mask=m)
|
||||
|
||||
|
||||
def sparse_buffers(
|
||||
*,
|
||||
c4_topk_lengths_clamp1: torch.Tensor,
|
||||
c4_topk_lengths_raw: torch.Tensor,
|
||||
c1_topk_lengths_clamp1: Optional[torch.Tensor],
|
||||
c2_topk_lengths_clamp1: Optional[torch.Tensor],
|
||||
index_topk: int,
|
||||
page_index_align: int,
|
||||
) -> Dict[str, torch.Tensor]:
|
||||
"""init_flashmla_related's tensors in one launch, keyed by attribute name: the top-k
|
||||
lengths clamped to index_topk and the -1-filled int32 page-index buffers padded to
|
||||
page_index_align, for ratio 4 and the ratios whose clamp-1 lengths are given."""
|
||||
for t in (
|
||||
c4_topk_lengths_clamp1,
|
||||
c4_topk_lengths_raw,
|
||||
c1_topk_lengths_clamp1,
|
||||
c2_topk_lengths_clamp1,
|
||||
):
|
||||
assert t is None or (
|
||||
t.dtype is torch.int32 and t.dim() == 1 and t.is_contiguous()
|
||||
), t
|
||||
rows = c4_topk_lengths_clamp1.shape[0]
|
||||
assert c4_topk_lengths_raw.shape[0] == rows
|
||||
device = c4_topk_lengths_clamp1.device
|
||||
width = triton.cdiv(index_topk, page_index_align) * page_index_align
|
||||
i32 = dict(dtype=torch.int32, device=device)
|
||||
out = {
|
||||
"c4_sparse_topk_lengths": torch.empty(rows, **i32),
|
||||
"c4_sparse_topk_lengths_raw": torch.empty(rows, **i32),
|
||||
"c4_sparse_page_indices": torch.empty((rows, width), **i32),
|
||||
}
|
||||
has_c1 = c1_topk_lengths_clamp1 is not None
|
||||
has_c2 = c2_topk_lengths_clamp1 is not None
|
||||
for ratio, has in ((1, has_c1), (2, has_c2)):
|
||||
if has:
|
||||
out[f"c{ratio}_sparse_topk_lengths"] = torch.empty(rows, **i32)
|
||||
out[f"c{ratio}_sparse_page_indices"] = torch.empty((rows, width), **i32)
|
||||
if rows == 0:
|
||||
return out
|
||||
placeholder = out["c4_sparse_topk_lengths"]
|
||||
fill_block = 1024
|
||||
per_buffer = triton.cdiv(rows * width, fill_block)
|
||||
n_buffers = 1 + int(has_c1) + int(has_c2)
|
||||
_sparse_buffers_kernel[(1 + per_buffer * n_buffers,)](
|
||||
c4_topk_lengths_clamp1,
|
||||
c4_topk_lengths_raw,
|
||||
out["c4_sparse_topk_lengths"],
|
||||
out["c4_sparse_topk_lengths_raw"],
|
||||
out["c4_sparse_page_indices"],
|
||||
c1_topk_lengths_clamp1 if has_c1 else placeholder,
|
||||
out.get("c1_sparse_topk_lengths", placeholder),
|
||||
out.get("c1_sparse_page_indices", placeholder),
|
||||
c2_topk_lengths_clamp1 if has_c2 else placeholder,
|
||||
out.get("c2_sparse_topk_lengths", placeholder),
|
||||
out.get("c2_sparse_page_indices", placeholder),
|
||||
rows,
|
||||
index_topk,
|
||||
width,
|
||||
HAS_C1=has_c1,
|
||||
HAS_C2=has_c2,
|
||||
BLOCK=1024,
|
||||
FILL_BLOCK=fill_block,
|
||||
num_warps=4,
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _page_table_from_req_to_token_kernel(
|
||||
r2t_ptr,
|
||||
req_ptr,
|
||||
out_ptr,
|
||||
num_cols,
|
||||
r2t_stride,
|
||||
PAGE: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0)
|
||||
blk = tl.program_id(1)
|
||||
req = tl.load(req_ptr + row).to(tl.int64)
|
||||
col = blk * BLOCK + tl.arange(0, BLOCK)
|
||||
mask = col < num_cols
|
||||
v = tl.load(
|
||||
r2t_ptr + req * r2t_stride + col.to(tl.int64) * PAGE, mask=mask, other=0
|
||||
)
|
||||
# torch's floor division; the C-style quotient rounds toward zero
|
||||
q = v // PAGE
|
||||
q = tl.where((v < 0) & (v % PAGE != 0), q - 1, q)
|
||||
tl.store(out_ptr + row.to(tl.int64) * num_cols + col, q.to(tl.int32), mask=mask)
|
||||
|
||||
|
||||
def page_table_from_req_to_token(
|
||||
req_to_token: torch.Tensor,
|
||||
req_rows: torch.Tensor,
|
||||
max_seq_len: int,
|
||||
page_size: int,
|
||||
) -> torch.Tensor:
|
||||
"""(req_to_token[req_rows, :max_seq_len:page_size] // page_size).to(torch.int32) in one
|
||||
launch (the gather, the floor division and the cast)."""
|
||||
assert req_to_token.dim() == 2 and req_to_token.stride(1) == 1
|
||||
assert req_rows.dim() == 1 and req_rows.stride(0) == 1
|
||||
n = req_rows.shape[0]
|
||||
num_cols = max(
|
||||
0, (min(max_seq_len, req_to_token.shape[1]) + page_size - 1) // page_size
|
||||
)
|
||||
out = torch.empty((n, num_cols), dtype=torch.int32, device=req_to_token.device)
|
||||
if n == 0 or num_cols == 0:
|
||||
return out
|
||||
block = min(1024, triton.next_power_of_2(num_cols))
|
||||
_page_table_from_req_to_token_kernel[(n, triton.cdiv(num_cols, block))](
|
||||
req_to_token,
|
||||
req_rows,
|
||||
out,
|
||||
num_cols,
|
||||
req_to_token.stride(0),
|
||||
PAGE=page_size,
|
||||
BLOCK=block,
|
||||
num_warps=4,
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _widen_pair_kernel(a_ptr, b_ptr, oa_ptr, ob_ptr, n, BLOCK: tl.constexpr):
|
||||
i = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)
|
||||
mask = i < n
|
||||
tl.store(oa_ptr + i, tl.load(a_ptr + i, mask=mask, other=0).to(tl.int64), mask=mask)
|
||||
tl.store(ob_ptr + i, tl.load(b_ptr + i, mask=mask, other=0).to(tl.int64), mask=mask)
|
||||
|
||||
|
||||
def widen_pair_i64(
|
||||
a: torch.Tensor, b: torch.Tensor
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
"""(a.to(torch.int64), b.to(torch.int64)) for two same-length integer vectors in one
|
||||
launch (the decode request slots and positions the low-ratio indexer takes widened)."""
|
||||
assert a.dim() == b.dim() == 1 and a.shape == b.shape, (a.shape, b.shape)
|
||||
assert a.stride(0) == 1 and b.stride(0) == 1
|
||||
n = a.shape[0]
|
||||
oa = torch.empty(n, dtype=torch.int64, device=a.device)
|
||||
ob = torch.empty(n, dtype=torch.int64, device=a.device)
|
||||
if n:
|
||||
block = min(1024, triton.next_power_of_2(n))
|
||||
_widen_pair_kernel[(triton.cdiv(n, block),)](
|
||||
a, b, oa, ob, n, BLOCK=block, num_warps=4
|
||||
)
|
||||
return oa, ob
|
||||
@@ -0,0 +1,408 @@
|
||||
"""Candidate blocks of the DeepSeek V4.1 two-level indexer on ROCm: level-one block scores and
|
||||
top blocks, the top-k within them, and the sorted paged top-k (the HIP side of candidate_blocks)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import List, NamedTuple, Optional
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
from sglang.kernels.ops.attention.dsv4 import topk_transform_paged
|
||||
|
||||
# the k the AOT fast_topk op (topk_hip.hip) is instantiated for; any other k takes torch.topk
|
||||
_AOT_FAST_TOPK_K = 2048
|
||||
|
||||
|
||||
def topk_transform_paged_sorted(
|
||||
scores: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
page_table: torch.Tensor,
|
||||
page_indices: torch.Tensor,
|
||||
page_size: int,
|
||||
raw_indices: Optional[torch.Tensor],
|
||||
) -> None:
|
||||
"""topk_transform_paged with every row sorted ascending by position in the AOT
|
||||
kernel's epilogue, -1 padding last."""
|
||||
torch.ops.sgl_kernel.deepseek_v4_topk_transform_512(
|
||||
scores, seq_lens, page_table, page_indices, page_size, raw_indices, True
|
||||
)
|
||||
|
||||
|
||||
# Candidate blocks one Triton program reduces; times block_size positions of logits.
|
||||
_LEVEL_ONE_BLOCKS_PER_PROGRAM = 256
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _candidate_block_scores_kernel(
|
||||
logits_ptr,
|
||||
seq_lens_ptr,
|
||||
out_ptr,
|
||||
logits_stride,
|
||||
width,
|
||||
num_blocks,
|
||||
out_stride,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
BLOCKS_PER_PROGRAM: tl.constexpr,
|
||||
FILL_TAIL: tl.constexpr,
|
||||
):
|
||||
"""out[row, blk] = max(logits[row, blk * BLOCK_SIZE : (blk + 1) * BLOCK_SIZE])
|
||||
over the positions < seq_lens[row]; +inf for the block holding the newest
|
||||
position, -inf for blocks past the reach (written only with FILL_TAIL). A
|
||||
program whose blocks all lie past the reach reads no logits."""
|
||||
row = tl.program_id(0)
|
||||
block0 = tl.program_id(1) * BLOCKS_PER_PROGRAM
|
||||
length = tl.load(seq_lens_ptr + row)
|
||||
blocks = block0 + tl.arange(0, BLOCKS_PER_PROGRAM)
|
||||
in_table = blocks < num_blocks
|
||||
if block0 * BLOCK_SIZE >= length:
|
||||
if FILL_TAIL:
|
||||
tl.store(out_ptr + row * out_stride + blocks, float("-inf"), mask=in_table)
|
||||
return
|
||||
cols = blocks[:, None] * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)[None, :]
|
||||
vals = tl.load(
|
||||
logits_ptr + row * logits_stride + cols,
|
||||
mask=(cols < length) & (cols < width),
|
||||
other=float("-inf"),
|
||||
)
|
||||
scores = tl.max(vals, axis=1)
|
||||
last = (length - 1) // BLOCK_SIZE
|
||||
scores = tl.where(blocks == last, float("inf"), scores)
|
||||
tl.store(out_ptr + row * out_stride + blocks, scores, mask=in_table)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _gather_candidate_blocks_kernel(
|
||||
logits_ptr,
|
||||
seq_lens_ptr,
|
||||
ids_ptr,
|
||||
out_ptr,
|
||||
logits_stride,
|
||||
width,
|
||||
ids_stride,
|
||||
out_stride,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
BLOCKS_PER_PROGRAM: tl.constexpr,
|
||||
):
|
||||
"""out[row, j * BLOCK_SIZE + t] = logits[row, ids[row, j] * BLOCK_SIZE + t] for
|
||||
the reachable positions of the kept blocks, -inf elsewhere (an id of -1, or a
|
||||
position past the reach in the newest block)."""
|
||||
row = tl.program_id(0)
|
||||
j0 = tl.program_id(1) * BLOCKS_PER_PROGRAM
|
||||
length = tl.load(seq_lens_ptr + row)
|
||||
j = j0 + tl.arange(0, BLOCKS_PER_PROGRAM)
|
||||
ids = tl.load(ids_ptr + row * ids_stride + j)
|
||||
t = tl.arange(0, BLOCK_SIZE)[None, :]
|
||||
cols = ids[:, None] * BLOCK_SIZE + t
|
||||
vals = tl.load(
|
||||
logits_ptr + row * logits_stride + cols,
|
||||
mask=(ids[:, None] >= 0) & (cols < length) & (cols < width),
|
||||
other=float("-inf"),
|
||||
)
|
||||
tl.store(out_ptr + row * out_stride + j[:, None] * BLOCK_SIZE + t, vals)
|
||||
|
||||
|
||||
def candidate_block_scores(
|
||||
logits: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
*,
|
||||
block_size: int,
|
||||
fill_tail: bool,
|
||||
) -> torch.Tensor:
|
||||
"""[rows, num_blocks] fp32 block maxima of logits over the reachable positions
|
||||
(see _candidate_block_scores_kernel). seq_lens int32 [rows], contiguous."""
|
||||
assert logits.dim() == 2 and logits.dtype == torch.float32 and logits.stride(1) == 1
|
||||
assert block_size & (block_size - 1) == 0, f"{block_size = } must be a power of 2"
|
||||
rows, width = logits.shape
|
||||
num_blocks = triton.cdiv(width, block_size)
|
||||
scores = torch.empty((rows, num_blocks), dtype=torch.float32, device=logits.device)
|
||||
grid = (rows, triton.cdiv(num_blocks, _LEVEL_ONE_BLOCKS_PER_PROGRAM))
|
||||
_candidate_block_scores_kernel[grid](
|
||||
logits,
|
||||
seq_lens,
|
||||
scores,
|
||||
logits.stride(0),
|
||||
width,
|
||||
num_blocks,
|
||||
scores.stride(0),
|
||||
BLOCK_SIZE=block_size,
|
||||
BLOCKS_PER_PROGRAM=_LEVEL_ONE_BLOCKS_PER_PROGRAM,
|
||||
FILL_TAIL=fill_tail,
|
||||
)
|
||||
return scores
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _candidate_lengths_kernel(
|
||||
seq_lens_ptr,
|
||||
block_lens_ptr,
|
||||
compact_lens_ptr,
|
||||
rows,
|
||||
topk_blocks,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
"""block_lens = ceil(len / BLOCK_SIZE): the blocks with a reachable position;
|
||||
compact_lens = min(block_lens, topk_blocks) * BLOCK_SIZE: the width of the
|
||||
compact candidate row (every block is kept while there are at most
|
||||
topk_blocks of them)."""
|
||||
r = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)
|
||||
mask = r < rows
|
||||
length = tl.load(seq_lens_ptr + r, mask=mask, other=0)
|
||||
block_lens = (length + (BLOCK_SIZE - 1)) // BLOCK_SIZE
|
||||
tl.store(block_lens_ptr + r, block_lens, mask=mask)
|
||||
tl.store(
|
||||
compact_lens_ptr + r,
|
||||
tl.minimum(block_lens, topk_blocks) * BLOCK_SIZE,
|
||||
mask=mask,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _map_compact_selection_kernel(
|
||||
compact_pos_ptr,
|
||||
ids_ptr,
|
||||
seq_lens_ptr,
|
||||
page_table_ptr,
|
||||
page_indices_ptr,
|
||||
raw_indices_ptr,
|
||||
ids_stride,
|
||||
pt_stride,
|
||||
out_stride,
|
||||
n_pages,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
PAGE_SIZE: tl.constexpr,
|
||||
TOPK: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
WRITE_RAW: tl.constexpr,
|
||||
SORT: tl.constexpr,
|
||||
PAD_KEY: tl.constexpr,
|
||||
):
|
||||
"""Compact position c -> real position ids[c // BLOCK_SIZE] * BLOCK_SIZE + c % BLOCK_SIZE ->
|
||||
slot via the row's page table; reachable selections packed first, -1 after. With SORT the valid
|
||||
prefix is ascending by (position, slot)."""
|
||||
row = tl.program_id(0)
|
||||
length = tl.load(seq_lens_ptr + row)
|
||||
j = tl.arange(0, BLOCK)
|
||||
in_k = j < TOPK
|
||||
c = tl.load(compact_pos_ptr + row * TOPK + j, mask=in_k, other=-1)
|
||||
valid = c >= 0
|
||||
cc = tl.where(valid, c, 0)
|
||||
blk = tl.load(ids_ptr + row * ids_stride + cc // BLOCK_SIZE)
|
||||
real = blk * BLOCK_SIZE + cc % BLOCK_SIZE
|
||||
valid = valid & (blk >= 0) & (real < length)
|
||||
page = tl.load(
|
||||
page_table_ptr + row * pt_stride + tl.minimum(real // PAGE_SIZE, n_pages - 1),
|
||||
mask=valid,
|
||||
other=0,
|
||||
)
|
||||
slot = page * PAGE_SIZE + real % PAGE_SIZE
|
||||
if SORT:
|
||||
# BLOCK == TOPK (power of two): the sort key is the position with raw indices, else the slot
|
||||
if WRITE_RAW:
|
||||
hi = real
|
||||
else:
|
||||
hi = slot
|
||||
key = tl.where(valid, hi, PAD_KEY).to(tl.int64) << 32
|
||||
key = tl.sort(
|
||||
key | (tl.where(valid, slot, -1).to(tl.int64) & 0xFFFFFFFF), dim=0
|
||||
)
|
||||
pad = (key >> 32) == PAD_KEY
|
||||
tl.store(
|
||||
page_indices_ptr + row * out_stride + j,
|
||||
tl.where(pad, -1, (key & 0xFFFFFFFF).to(tl.int32)),
|
||||
)
|
||||
if WRITE_RAW:
|
||||
tl.store(
|
||||
raw_indices_ptr + row * out_stride + j,
|
||||
tl.where(pad, -1, (key >> 32).to(tl.int32)),
|
||||
)
|
||||
else:
|
||||
v = valid.to(tl.int32)
|
||||
count = tl.sum(v, axis=0)
|
||||
wpos = tl.cumsum(v, axis=0) - 1
|
||||
# The packed entries and the padding never share an address.
|
||||
tl.store(page_indices_ptr + row * out_stride + j, -1, mask=in_k & (j >= count))
|
||||
tl.store(page_indices_ptr + row * out_stride + wpos, slot, mask=valid)
|
||||
if WRITE_RAW:
|
||||
tl.store(
|
||||
raw_indices_ptr + row * out_stride + j, -1, mask=in_k & (j >= count)
|
||||
)
|
||||
tl.store(raw_indices_ptr + row * out_stride + wpos, real, mask=valid)
|
||||
|
||||
|
||||
class CandidateBlocks(NamedTuple):
|
||||
"""What the candidate-source layer publishes for the decode rows of a step."""
|
||||
|
||||
# int32 [rows, topk_blocks]: kept block ids in no particular order, -1 padded after the last
|
||||
ids: torch.Tensor
|
||||
# int32 [rows]: kept blocks * block_size, the width of the compact row.
|
||||
compact_lens: torch.Tensor
|
||||
# int32 [rows, 1] zeros; with page_size = compact_page_size a position maps to itself
|
||||
compact_page_table: torch.Tensor
|
||||
compact_page_size: int
|
||||
block_size: int
|
||||
|
||||
|
||||
def slice_candidate_blocks(candidates: CandidateBlocks, rows: slice) -> CandidateBlocks:
|
||||
"""The rows rows of a per-request publication."""
|
||||
return candidates._replace(
|
||||
ids=candidates.ids[rows],
|
||||
compact_lens=candidates.compact_lens[rows],
|
||||
compact_page_table=candidates.compact_page_table[rows],
|
||||
)
|
||||
|
||||
|
||||
def cat_candidate_blocks(pieces: List[CandidateBlocks]) -> CandidateBlocks:
|
||||
"""Row chunks of one request's publication, in row order."""
|
||||
if len(pieces) == 1:
|
||||
return pieces[0]
|
||||
return pieces[0]._replace(
|
||||
ids=torch.cat([p.ids for p in pieces]),
|
||||
compact_lens=torch.cat([p.compact_lens for p in pieces]),
|
||||
compact_page_table=torch.cat([p.compact_page_table for p in pieces]),
|
||||
)
|
||||
|
||||
|
||||
def select_candidate_blocks_hip(
|
||||
logits: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
*,
|
||||
topk_blocks: int,
|
||||
block_size: int,
|
||||
) -> CandidateBlocks:
|
||||
"""Level one over a [rows, capacity] logits rectangle, bounded by seq_lens: the topk_blocks
|
||||
best blocks of each row, the block with its newest position always among them; as a set per
|
||||
row the ids equal the reference's select_candidate_blocks. Graph-safe: no host sync."""
|
||||
seq_lens = seq_lens.to(torch.int32).contiguous()
|
||||
rows, width = logits.shape
|
||||
device = logits.device
|
||||
num_blocks = triton.cdiv(width, block_size)
|
||||
use_aot = topk_blocks == _AOT_FAST_TOPK_K
|
||||
scores = candidate_block_scores(
|
||||
logits, seq_lens, block_size=block_size, fill_tail=not use_aot
|
||||
)
|
||||
block_lens = torch.empty(rows, dtype=torch.int32, device=device)
|
||||
compact_lens = torch.empty(rows, dtype=torch.int32, device=device)
|
||||
_candidate_lengths_kernel[(triton.cdiv(rows, 1024),)](
|
||||
seq_lens,
|
||||
block_lens,
|
||||
compact_lens,
|
||||
rows,
|
||||
topk_blocks,
|
||||
BLOCK_SIZE=block_size,
|
||||
BLOCK=1024,
|
||||
)
|
||||
if use_aot:
|
||||
# exact top-k over the first ceil(len / block_size) block scores of each row, -1 padded
|
||||
ids = torch.empty((rows, topk_blocks), dtype=torch.int32, device=device)
|
||||
torch.ops.sgl_kernel.fast_topk(scores, ids, block_lens, None)
|
||||
else:
|
||||
picked = scores.topk(min(topk_blocks, num_blocks), dim=-1)
|
||||
ids = picked.indices.to(torch.int32).masked_fill(
|
||||
picked.values == -torch.inf, -1
|
||||
)
|
||||
# the gather takes the width from ids and tiles it in 256-block programs
|
||||
ids = F.pad(ids, (0, topk_blocks - ids.shape[1]), value=-1)
|
||||
compact_width = topk_blocks * block_size
|
||||
return CandidateBlocks(
|
||||
ids=ids,
|
||||
compact_lens=compact_lens,
|
||||
compact_page_table=torch.zeros((rows, 1), dtype=torch.int32, device=device),
|
||||
compact_page_size=triton.next_power_of_2(compact_width),
|
||||
block_size=block_size,
|
||||
)
|
||||
|
||||
|
||||
def gather_candidate_blocks(
|
||||
logits: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
ids: torch.Tensor,
|
||||
*,
|
||||
block_size: int,
|
||||
) -> torch.Tensor:
|
||||
"""The compact [rows, topk_blocks * block_size] fp32 row of each request's
|
||||
candidate positions in id order (see _gather_candidate_blocks_kernel)."""
|
||||
rows, width = logits.shape
|
||||
topk_blocks = ids.shape[1]
|
||||
assert ids.shape[0] == rows and ids.dtype == torch.int32 and ids.stride(1) == 1
|
||||
per_program = min(topk_blocks, _LEVEL_ONE_BLOCKS_PER_PROGRAM)
|
||||
assert topk_blocks % per_program == 0, topk_blocks
|
||||
compact = torch.empty(
|
||||
(rows, topk_blocks * block_size), dtype=torch.float32, device=logits.device
|
||||
)
|
||||
grid = (rows, topk_blocks // per_program)
|
||||
_gather_candidate_blocks_kernel[grid](
|
||||
logits,
|
||||
seq_lens.to(torch.int32).contiguous(),
|
||||
ids,
|
||||
compact,
|
||||
logits.stride(0),
|
||||
width,
|
||||
ids.stride(0),
|
||||
compact.stride(0),
|
||||
BLOCK_SIZE=block_size,
|
||||
BLOCKS_PER_PROGRAM=per_program,
|
||||
)
|
||||
return compact
|
||||
|
||||
|
||||
def topk_within_candidate_blocks_hip(
|
||||
logits: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
candidates: CandidateBlocks,
|
||||
*,
|
||||
page_table: torch.Tensor,
|
||||
page_size: int,
|
||||
page_indices: torch.Tensor,
|
||||
raw_indices: Optional[torch.Tensor],
|
||||
sort_output: bool = False,
|
||||
) -> None:
|
||||
"""Level two for a consumer layer: the top-k of logits inside the published candidate
|
||||
blocks, written as the paged transform writes it (-1 padded, valid prefix first; ascending with
|
||||
sort_output, k a power of two). Runs on the compact row, so the cost stops growing with context."""
|
||||
rows, width = logits.shape
|
||||
topk = page_indices.shape[1]
|
||||
block_size = candidates.block_size
|
||||
seq_lens = seq_lens.to(torch.int32).contiguous()
|
||||
compact = gather_candidate_blocks(
|
||||
logits, seq_lens, candidates.ids, block_size=block_size
|
||||
)
|
||||
assert compact.shape[1] <= candidates.compact_page_size
|
||||
compact_pos = torch.empty((rows, topk), dtype=torch.int32, device=logits.device)
|
||||
topk_transform_paged(
|
||||
compact,
|
||||
candidates.compact_lens,
|
||||
candidates.compact_page_table,
|
||||
compact_pos,
|
||||
candidates.compact_page_size,
|
||||
None,
|
||||
)
|
||||
assert page_indices.stride(1) == 1 and page_indices.shape == (rows, topk)
|
||||
assert page_table.stride(1) == 1 and page_table.shape[0] == rows
|
||||
write_raw = raw_indices is not None
|
||||
if write_raw:
|
||||
assert raw_indices.shape == (rows, topk) and raw_indices.stride(1) == 1
|
||||
assert raw_indices.stride(0) == page_indices.stride(0)
|
||||
assert not sort_output or topk & (topk - 1) == 0, topk
|
||||
_map_compact_selection_kernel[(rows,)](
|
||||
compact_pos,
|
||||
candidates.ids,
|
||||
seq_lens,
|
||||
page_table,
|
||||
page_indices,
|
||||
raw_indices if write_raw else page_indices,
|
||||
candidates.ids.stride(0),
|
||||
page_table.stride(0),
|
||||
page_indices.stride(0),
|
||||
page_table.shape[1],
|
||||
BLOCK_SIZE=block_size,
|
||||
PAGE_SIZE=page_size,
|
||||
TOPK=topk,
|
||||
BLOCK=triton.next_power_of_2(topk),
|
||||
WRITE_RAW=write_raw,
|
||||
SORT=sort_output,
|
||||
PAD_KEY=torch.iinfo(torch.int32).max,
|
||||
)
|
||||
@@ -0,0 +1,403 @@
|
||||
"""Sparse attention reading V4.1 FP8/FP4 pages directly on HIP."""
|
||||
|
||||
from typing import Optional, Tuple
|
||||
|
||||
import torch
|
||||
import triton
|
||||
from aiter.ops.triton._gluon_kernels.gfx950.attention.pa_decode_sparse import _qkpv_fp8
|
||||
from triton.experimental import gluon
|
||||
from triton.experimental.gluon import language as gl
|
||||
|
||||
from sglang.kernels.ops.attention.dsv4.kv_layout import KVLayout
|
||||
from sglang.kernels.ops.attention.sparse_decode_reduce_hip import (
|
||||
gfx_sparse_split_reduce,
|
||||
)
|
||||
|
||||
RCP_LN2 = gl.constexpr(1.4426950408889634) # exp(x) = exp2(x * RCP_LN2)
|
||||
TILE_SLOTS = 64
|
||||
MAX_SPLITS = 8
|
||||
PACKED_TOKEN_BYTES = (KVLayout.V41.bytes_per_token, KVLayout.V41_FP4.bytes_per_token)
|
||||
|
||||
|
||||
@gluon.jit
|
||||
def _load_compact(Cache, offset, valid, USE_BUFFER: gl.constexpr):
|
||||
if USE_BUFFER:
|
||||
return gl.amd.cdna4.buffer_load(Cache, offset.to(gl.int32), valid, other=0)
|
||||
return gl.load(Cache + offset, valid, 0)
|
||||
|
||||
|
||||
@gluon.jit
|
||||
def _read_compact_gluon(
|
||||
Cache,
|
||||
slots,
|
||||
valid,
|
||||
PAGE: gl.constexpr,
|
||||
STRIDE: gl.constexpr,
|
||||
FP4: gl.constexpr,
|
||||
USE_BUFFER: gl.constexpr,
|
||||
G: gl.constexpr,
|
||||
):
|
||||
d = gl.arange(0, 512, layout=gl.SliceLayout(0, G))
|
||||
base = (slots // PAGE).to(gl.int64) * STRIDE
|
||||
row = slots % PAGE
|
||||
if FP4:
|
||||
PG: gl.constexpr = gl.BlockedLayout([1, 8], [8, 8], [1, 4], [1, 0])
|
||||
pd = gl.arange(0, 256, layout=gl.SliceLayout(0, PG))
|
||||
packed_base = gl.convert_layout(base, gl.SliceLayout(1, PG))
|
||||
packed_row = gl.convert_layout(row, gl.SliceLayout(1, PG))
|
||||
packed_valid = gl.convert_layout(valid, gl.SliceLayout(1, PG))
|
||||
packed = _load_compact(
|
||||
Cache,
|
||||
packed_base[:, None] + packed_row[:, None] * 256 + pd[None, :],
|
||||
packed_valid[:, None],
|
||||
USE_BUFFER,
|
||||
)
|
||||
value = gl.amd.cdna4.scaled_upcast(
|
||||
packed,
|
||||
gl.full((slots.shape[0], 512), 127, gl.uint8, G),
|
||||
gl.bfloat16,
|
||||
axis=1,
|
||||
).to(gl.float32)
|
||||
scale = (
|
||||
_load_compact(
|
||||
Cache,
|
||||
base[:, None] + PAGE * 256 + row[:, None] * 32 + d[None, :] // 16,
|
||||
valid[:, None],
|
||||
USE_BUFFER,
|
||||
)
|
||||
.to(gl.float8e4nv, bitcast=True)
|
||||
.to(gl.float32)
|
||||
)
|
||||
else:
|
||||
value = (
|
||||
_load_compact(
|
||||
Cache,
|
||||
base[:, None] + row[:, None] * 512 + d[None, :],
|
||||
valid[:, None],
|
||||
USE_BUFFER,
|
||||
)
|
||||
.to(gl.float8e4nv, bitcast=True)
|
||||
.to(gl.float32)
|
||||
)
|
||||
exp = _load_compact(
|
||||
Cache,
|
||||
base[:, None] + PAGE * 512 + row[:, None] * 16 + d[None, :] // 32,
|
||||
valid[:, None],
|
||||
USE_BUFFER,
|
||||
).to(gl.int32)
|
||||
scale = gl.where(exp == 0, 0x00400000, exp << 23).to(gl.float32, bitcast=True)
|
||||
scale = gl.where(exp == 255, float("nan"), scale)
|
||||
return (value * scale).to(gl.bfloat16)
|
||||
|
||||
|
||||
@gluon.jit
|
||||
def _compact_qkpv(
|
||||
kv,
|
||||
valid,
|
||||
qdot,
|
||||
m,
|
||||
den,
|
||||
acc,
|
||||
head_mask,
|
||||
smem,
|
||||
MF: gl.constexpr,
|
||||
H: gl.constexpr,
|
||||
BLOCK: gl.constexpr,
|
||||
SCALE: gl.constexpr,
|
||||
):
|
||||
# aiter's private helper, by keyword: a renamed or reordered parameter fails to compile
|
||||
return _qkpv_fp8(
|
||||
k_nope=kv,
|
||||
k_rope=kv,
|
||||
valid=valid,
|
||||
q_dot=qdot,
|
||||
m_i=m,
|
||||
l_i=den,
|
||||
acc=acc,
|
||||
head_mask=head_mask,
|
||||
qk_scale=SCALE * RCP_LN2,
|
||||
kv_smem=smem,
|
||||
qk_layout=MF,
|
||||
pv_layout=MF,
|
||||
k_layout=gl.DotOperandLayout(1, MF, 8),
|
||||
v_layout=gl.DotOperandLayout(1, MF, 8),
|
||||
p_layout=gl.DotOperandLayout(0, MF, 8),
|
||||
NOPE_DIM=512,
|
||||
ROPE_DIM=0,
|
||||
HEAD_SIZE=512,
|
||||
BLOCK_M=16,
|
||||
BLOCK_K=BLOCK,
|
||||
HEAD_ALIGNED=H % 16 == 0,
|
||||
UNIFORM=True,
|
||||
HAS_INVALID=True,
|
||||
)
|
||||
|
||||
|
||||
@gluon.jit
|
||||
def _compact_tile(
|
||||
Cache,
|
||||
Indices,
|
||||
t,
|
||||
start,
|
||||
length,
|
||||
STRIDE: gl.constexpr,
|
||||
CAPACITY: gl.constexpr,
|
||||
PAGE: gl.constexpr,
|
||||
CACHE_STRIDE: gl.constexpr,
|
||||
FP4: gl.constexpr,
|
||||
BUFFER: gl.constexpr,
|
||||
G: gl.constexpr,
|
||||
BLOCK: gl.constexpr,
|
||||
):
|
||||
col = start + gl.arange(0, BLOCK, layout=gl.SliceLayout(1, G))
|
||||
slot = gl.load(Indices + t * STRIDE + col, col < length, -1)
|
||||
valid = (col < length) & (slot >= 0) & (slot < CAPACITY)
|
||||
kv = _read_compact_gluon(
|
||||
Cache, gl.where(valid, slot, 0), valid, PAGE, CACHE_STRIDE, FP4, BUFFER, G
|
||||
)
|
||||
return kv, valid
|
||||
|
||||
|
||||
@gluon.jit
|
||||
def _compact_attention_kernel(
|
||||
Q,
|
||||
K,
|
||||
E,
|
||||
I,
|
||||
EI,
|
||||
L,
|
||||
EL,
|
||||
Acc,
|
||||
Max,
|
||||
Sum,
|
||||
Out,
|
||||
Sink,
|
||||
Freqs,
|
||||
Positions,
|
||||
HAS_SINK: gl.constexpr,
|
||||
ROPE_DIM: gl.constexpr,
|
||||
FREQ_STRIDE: gl.constexpr,
|
||||
QS: gl.constexpr,
|
||||
QH: gl.constexpr,
|
||||
H: gl.constexpr,
|
||||
KS: gl.constexpr,
|
||||
ES: gl.constexpr,
|
||||
KP: gl.constexpr,
|
||||
EP: gl.constexpr,
|
||||
KN: gl.constexpr,
|
||||
EN: gl.constexpr,
|
||||
IS: gl.constexpr,
|
||||
EIS: gl.constexpr,
|
||||
NK: gl.constexpr,
|
||||
NE: gl.constexpr,
|
||||
KFP4: gl.constexpr,
|
||||
EFP4: gl.constexpr,
|
||||
KBUFFER: gl.constexpr,
|
||||
EBUFFER: gl.constexpr,
|
||||
SPLITS: gl.constexpr,
|
||||
SCALE: gl.constexpr,
|
||||
BLOCK: gl.constexpr,
|
||||
):
|
||||
t, split, hg = gl.program_id(0).to(gl.int64), gl.program_id(1), gl.program_id(2)
|
||||
MF: gl.constexpr = gl.amd.AMDMFMALayout(
|
||||
version=4, instr_shape=[16, 16, 16], transposed=True, warps_per_cta=[1, 4]
|
||||
)
|
||||
G: gl.constexpr = gl.BlockedLayout([1, 16], [8, 8], [1, 4], [1, 0])
|
||||
QG: gl.constexpr = gl.BlockedLayout([1, 8], [8, 8], [1, 4], [1, 0])
|
||||
hq = hg * 16 + gl.arange(0, 16, layout=gl.SliceLayout(1, QG))
|
||||
dq = gl.arange(0, 512, layout=gl.SliceLayout(0, QG))
|
||||
q = gl.load(Q + t * QS + hq[:, None] * QH + dq[None, :], hq[:, None] < H, 0)
|
||||
qdot = gl.convert_layout(q, gl.DotOperandLayout(0, MF, 8))
|
||||
h = hg * 16 + gl.arange(0, 16, layout=gl.SliceLayout(1, MF))
|
||||
d = gl.arange(0, 512, layout=gl.SliceLayout(0, MF))
|
||||
m = gl.full((16,), -float("inf"), gl.float32, layout=gl.SliceLayout(1, MF))
|
||||
den = gl.zeros((16,), gl.float32, layout=gl.SliceLayout(1, MF))
|
||||
acc = gl.zeros((16, 512), gl.float32, layout=MF)
|
||||
smem = gl.allocate_shared_memory(
|
||||
gl.bfloat16,
|
||||
[BLOCK, 512],
|
||||
gl.PaddedSharedLayout.with_identity_for([[512, 8]], [BLOCK, 512], [1, 0]),
|
||||
)
|
||||
for segment in gl.static_range(2):
|
||||
if segment == 0 or NE > 0:
|
||||
if segment == 0:
|
||||
indices, lengths, width, stride, capacity = I, L, NK, IS, KN
|
||||
cache, page, cache_stride, fp4, buffer = K, KP, KS, KFP4, KBUFFER
|
||||
else:
|
||||
indices, lengths, width, stride, capacity = EI, EL, NE, EIS, EN
|
||||
cache, page, cache_stride, fp4, buffer = E, EP, ES, EFP4, EBUFFER
|
||||
length = gl.minimum(gl.load(lengths + t), width)
|
||||
if split * BLOCK < length:
|
||||
kv, valid = _compact_tile(
|
||||
cache,
|
||||
indices,
|
||||
t,
|
||||
split * BLOCK,
|
||||
length,
|
||||
stride,
|
||||
capacity,
|
||||
page,
|
||||
cache_stride,
|
||||
fp4,
|
||||
buffer,
|
||||
G,
|
||||
BLOCK,
|
||||
)
|
||||
for start in range((split + SPLITS) * BLOCK, length, SPLITS * BLOCK):
|
||||
next_kv, next_valid = _compact_tile(
|
||||
cache,
|
||||
indices,
|
||||
t,
|
||||
start,
|
||||
length,
|
||||
stride,
|
||||
capacity,
|
||||
page,
|
||||
cache_stride,
|
||||
fp4,
|
||||
buffer,
|
||||
G,
|
||||
BLOCK,
|
||||
)
|
||||
m, den, acc = _compact_qkpv(
|
||||
kv, valid, qdot, m, den, acc, h < H, smem, MF, H, BLOCK, SCALE
|
||||
)
|
||||
kv, valid = next_kv, next_valid
|
||||
m, den, acc = _compact_qkpv(
|
||||
kv, valid, qdot, m, den, acc, h < H, smem, MF, H, BLOCK, SCALE
|
||||
)
|
||||
if SPLITS == 1:
|
||||
if HAS_SINK:
|
||||
sink = gl.load(Sink + h, h < H, -float("inf"))
|
||||
scaled_max = m * (SCALE * RCP_LN2)
|
||||
final_max = gl.maximum(scaled_max, sink * RCP_LN2)
|
||||
weight = gl.exp2(scaled_max - final_max)
|
||||
den = den * weight + gl.exp2(sink * RCP_LN2 - final_max)
|
||||
acc = acc * weight[:, None]
|
||||
out = (acc / den[:, None]).to(gl.bfloat16)
|
||||
out = gl.convert_layout(out, QG)
|
||||
if ROPE_DIM > 0:
|
||||
pos = gl.load(Positions + t)
|
||||
is_rope = dq >= 512 - ROPE_DIM
|
||||
freq_index = ((dq - (512 - ROPE_DIM)) // 2) * 2
|
||||
cos = gl.load(Freqs + pos * FREQ_STRIDE + freq_index, is_rope, 0)
|
||||
sin = gl.load(Freqs + pos * FREQ_STRIDE + freq_index + 1, is_rope, 0)
|
||||
x = out.to(gl.float32)
|
||||
even, odd = gl.split(gl.reshape(x, (16, 256, 2)))
|
||||
swapped = gl.reshape(gl.join(odd, -even), (16, 512))
|
||||
rotated = gl.fma(x, cos[None, :], swapped * sin[None, :])
|
||||
out = gl.where(is_rope[None, :], rotated.to(gl.bfloat16), out)
|
||||
gl.store(Out + (t * H + hq[:, None]) * 512 + dq[None, :], out, hq[:, None] < H)
|
||||
else:
|
||||
off = (t * SPLITS + split) * H + h
|
||||
gl.store(Max + off, m * (SCALE * RCP_LN2), h < H)
|
||||
gl.store(Sum + off, den, h < H)
|
||||
gl.store(Acc + off[:, None] * 512 + d[None, :], acc, h[:, None] < H)
|
||||
|
||||
|
||||
def compact_attention_hip(
|
||||
q: torch.Tensor,
|
||||
cache: torch.Tensor,
|
||||
indices: torch.Tensor,
|
||||
lengths: torch.Tensor,
|
||||
sink: Optional[torch.Tensor],
|
||||
*,
|
||||
extra_cache: Optional[torch.Tensor] = None,
|
||||
extra_indices: Optional[torch.Tensor] = None,
|
||||
extra_lengths: Optional[torch.Tensor] = None,
|
||||
softmax_scale: float = 512**-0.5,
|
||||
splits: Optional[int] = None,
|
||||
inv_rope: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
|
||||
) -> torch.Tensor:
|
||||
"""q [n, h, 512] bf16 over the first lengths[t] slots of indices[t] (-1 skipped) in the
|
||||
packed V41 / V41_FP4 cache [pages, page_size, 1, bytes_per_token], then those of
|
||||
extra_indices[t] in extra_cache, the sink folded in once. inv_rope is
|
||||
(freqs_real, positions): the tail of every head is inverse-rotated."""
|
||||
assert q.ndim == 3 and q.shape[-1] == 512 and q.dtype == torch.bfloat16
|
||||
assert q.stride(-1) == 1
|
||||
assert cache.ndim == 4 and cache.shape[2] == 1
|
||||
assert cache.shape[-1] in PACKED_TOKEN_BYTES
|
||||
n, h, _ = q.shape
|
||||
if n == 0:
|
||||
return torch.empty_like(q)
|
||||
indices = indices.reshape(n, -1)
|
||||
assert indices.stride(-1) == 1 and lengths.is_contiguous()
|
||||
if extra_cache is None:
|
||||
assert extra_indices is None and extra_lengths is None
|
||||
extra_cache, extra_indices, extra_lengths = cache, indices, lengths
|
||||
ne = 0
|
||||
else:
|
||||
assert extra_cache.shape[-1] in PACKED_TOKEN_BYTES
|
||||
extra_indices = extra_indices.reshape(n, -1)
|
||||
assert extra_indices.stride(-1) == 1 and extra_lengths.is_contiguous()
|
||||
ne = extra_indices.shape[1]
|
||||
if splits is None:
|
||||
# enough splits to fill the CUs, none past the slots a row has
|
||||
num_cus = torch.cuda.get_device_properties(q.device).multi_processor_count
|
||||
splits = min(
|
||||
MAX_SPLITS,
|
||||
triton.cdiv(indices.shape[1] + ne, TILE_SLOTS),
|
||||
triton.cdiv(num_cus, n * triton.cdiv(h, 16)),
|
||||
)
|
||||
assert splits > 0
|
||||
out = torch.empty((n, h, 512), dtype=q.dtype, device=q.device) if splits == 1 else q
|
||||
if inv_rope is not None:
|
||||
freqs, positions = inv_rope
|
||||
assert freqs.dtype == torch.float32 and freqs.stride(1) == 1
|
||||
assert positions.shape == (n,)
|
||||
rope_dim = freqs.shape[1]
|
||||
assert 0 < rope_dim <= 512 and rope_dim % 2 == 0
|
||||
else:
|
||||
freqs = positions = out
|
||||
rope_dim = 0
|
||||
if splits == 1:
|
||||
acc = maximum = denominator = out
|
||||
else:
|
||||
acc = torch.empty((n, splits, h, 512), dtype=torch.float32, device=q.device)
|
||||
maximum = torch.empty((n, splits, h), dtype=torch.float32, device=q.device)
|
||||
denominator = torch.empty_like(maximum)
|
||||
_compact_attention_kernel[(n, splits, triton.cdiv(h, 16))](
|
||||
q,
|
||||
cache.view(torch.uint8),
|
||||
extra_cache.view(torch.uint8),
|
||||
indices,
|
||||
extra_indices,
|
||||
lengths,
|
||||
extra_lengths,
|
||||
acc,
|
||||
maximum,
|
||||
denominator,
|
||||
out,
|
||||
sink if sink is not None else out,
|
||||
freqs,
|
||||
positions,
|
||||
HAS_SINK=sink is not None,
|
||||
ROPE_DIM=rope_dim,
|
||||
FREQ_STRIDE=freqs.stride(0),
|
||||
QS=q.stride(0),
|
||||
QH=q.stride(1),
|
||||
H=h,
|
||||
KS=cache.stride(0),
|
||||
ES=extra_cache.stride(0),
|
||||
KP=cache.shape[1],
|
||||
EP=extra_cache.shape[1],
|
||||
KN=cache.shape[0] * cache.shape[1],
|
||||
EN=extra_cache.shape[0] * extra_cache.shape[1],
|
||||
IS=indices.stride(0),
|
||||
EIS=extra_indices.stride(0),
|
||||
NK=indices.shape[1],
|
||||
NE=ne,
|
||||
KFP4=cache.shape[-1] == KVLayout.V41_FP4.bytes_per_token,
|
||||
EFP4=extra_cache.shape[-1] == KVLayout.V41_FP4.bytes_per_token,
|
||||
KBUFFER=cache.shape[0] * cache.stride(0) < 2**31,
|
||||
EBUFFER=extra_cache.shape[0] * extra_cache.stride(0) < 2**31,
|
||||
SPLITS=splits,
|
||||
SCALE=softmax_scale,
|
||||
BLOCK=TILE_SLOTS,
|
||||
num_warps=4,
|
||||
)
|
||||
if splits == 1:
|
||||
return out
|
||||
return gfx_sparse_split_reduce(
|
||||
acc, maximum, denominator, sink, q.dtype, inv_rope=inv_rope
|
||||
)
|
||||
@@ -1,5 +1,5 @@
|
||||
"""SM100 small-batch paged attention with the heads on the MMA N dimension; the
|
||||
caller applies the inverse RoPE to the result."""
|
||||
inverse RoPE is applied by the caller, or by the combine when inv_rope is given."""
|
||||
|
||||
from typing import Optional
|
||||
|
||||
@@ -37,6 +37,23 @@ def can_use_swapab_attention(
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def inverse_rope_tail(
|
||||
out, d, FREQS, pos, freq_stride, HEAD_DIM: tl.constexpr, ROPE_DIM: tl.constexpr
|
||||
):
|
||||
"""out (head dims d) with its last ROPE_DIM dims inverse-rotated at position pos. It
|
||||
rotates the rounded values, as the model's standalone RoPE kernel reads them."""
|
||||
x = out.to(tl.float32)
|
||||
is_rope = d >= HEAD_DIM - ROPE_DIM
|
||||
freq_index = ((d - (HEAD_DIM - ROPE_DIM)) // 2) * 2
|
||||
cos = tl.load(FREQS + pos * freq_stride + freq_index, is_rope, 0)
|
||||
sin = tl.load(FREQS + pos * freq_stride + freq_index + 1, is_rope, 0)
|
||||
signed = tl.where(d % 2 == 0, -x * sin, x * sin)
|
||||
swapped = tl.reshape(tl.flip(tl.reshape(signed, (d.shape[0] // 2, 2)), 1), d.shape)
|
||||
rotated = tl.fma(x, cos, swapped)
|
||||
return tl.where(is_rope, rotated.to(out.dtype), out)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _combine(
|
||||
PART,
|
||||
@@ -44,6 +61,10 @@ def _combine(
|
||||
SUM,
|
||||
SINK,
|
||||
OUT,
|
||||
FREQS,
|
||||
POSITIONS,
|
||||
FREQ_STRIDE: tl.constexpr,
|
||||
ROPE_DIM: tl.constexpr,
|
||||
NT: tl.constexpr,
|
||||
ST: tl.constexpr,
|
||||
H: tl.constexpr,
|
||||
@@ -66,6 +87,10 @@ def _combine(
|
||||
)
|
||||
out = tl.sum(vals * factor[:, None], 0) / denominator
|
||||
out = tl.where((denominator > 0) & (sink != float("inf")), out, 0.0)
|
||||
out = out.to(OUT.dtype.element_ty)
|
||||
if ROPE_DIM > 0:
|
||||
pos = tl.load(POSITIONS + b)
|
||||
out = inverse_rope_tail(out, d, FREQS, pos, FREQ_STRIDE, 512, ROPE_DIM)
|
||||
tl.store(OUT + (b * H + h) * 512 + d, out)
|
||||
|
||||
|
||||
@@ -78,10 +103,14 @@ def swapab_attention(
|
||||
extra_kv=None,
|
||||
extra_indices=None,
|
||||
extra_lengths=None,
|
||||
inv_rope=None,
|
||||
partial_gluon=None,
|
||||
):
|
||||
"""V4-layout attention on 16 heads; `extra_*` is a second slot range appended
|
||||
to each request's keys, and the attention sink is folded in exactly once."""
|
||||
from .decode_attention_sm100_gluon import partial_gluon
|
||||
to each request's keys, and the attention sink is folded in exactly once.
|
||||
`partial_gluon` replaces the SM100 split-KV kernel (swapab_gluon_hip passes gfx950's)."""
|
||||
if partial_gluon is None:
|
||||
from .decode_attention_sm100_gluon import partial_gluon
|
||||
|
||||
block = 64
|
||||
b, h, d = q.shape[0], q.shape[-2], q.shape[-1]
|
||||
@@ -144,6 +173,15 @@ def swapab_attention(
|
||||
TILE=LAYOUT.tile_size,
|
||||
num_warps=4,
|
||||
)
|
||||
if inv_rope is not None:
|
||||
freqs, positions = inv_rope
|
||||
assert freqs.dtype == torch.float32 and freqs.stride(1) == 1
|
||||
assert positions.shape == (b,)
|
||||
rope_dim = freqs.shape[1]
|
||||
assert 0 < rope_dim <= 512 and rope_dim % 2 == 0
|
||||
else:
|
||||
freqs = positions = out
|
||||
rope_dim = 0
|
||||
bd = 64 if ne else 512
|
||||
_combine[(b, h, triton.cdiv(512, bd))](
|
||||
partial,
|
||||
@@ -151,6 +189,10 @@ def swapab_attention(
|
||||
sums,
|
||||
sink,
|
||||
out,
|
||||
freqs,
|
||||
positions,
|
||||
FREQ_STRIDE=freqs.stride(0),
|
||||
ROPE_DIM=rope_dim,
|
||||
NT=nt,
|
||||
ST=triton.next_power_of_2(nt),
|
||||
H=h,
|
||||
|
||||
@@ -5,11 +5,19 @@ import triton
|
||||
import triton.language as tl
|
||||
|
||||
from sglang.kernels.jit.utils import is_arch_support_pdl
|
||||
from sglang.srt.utils import is_hip
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _q_rope_store(
|
||||
X, Y, F, POS, SX: tl.constexpr, SY: tl.constexpr, USE_GDC: tl.constexpr = False
|
||||
X,
|
||||
Y,
|
||||
F,
|
||||
POS,
|
||||
SX: tl.constexpr,
|
||||
SY: tl.constexpr,
|
||||
IS_HIP: tl.constexpr,
|
||||
USE_GDC: tl.constexpr = False,
|
||||
):
|
||||
row, head = tl.program_id(0), tl.program_id(1)
|
||||
r = tl.arange(0, 512)
|
||||
@@ -22,7 +30,11 @@ def _q_rope_store(
|
||||
sin = tl.load(F + position * 64 + (r - 448) // 2 * 2 + 1, r >= 448, 0)
|
||||
# Match the operation order of deepseek_rope_kernel before BF16 rounding.
|
||||
even = tl.fma(value, cos, -partner * sin)
|
||||
odd = tl.fma(partner, sin, value * cos)
|
||||
if IS_HIP:
|
||||
# Match AMD flat RoPE: round the sine product before the cosine FMA.
|
||||
odd = tl.fma(value, cos, partner * sin)
|
||||
else:
|
||||
odd = tl.fma(partner, sin, value * cos)
|
||||
rotated = tl.where((r & 1) == 0, even, odd)
|
||||
tl.store(Y + row * SY + head * 512 + r, tl.where(r >= 448, rotated, value))
|
||||
if USE_GDC:
|
||||
@@ -40,6 +52,7 @@ def _q_rope_store_prefill(
|
||||
SX: tl.constexpr,
|
||||
SY: tl.constexpr,
|
||||
BLOCK_HEADS: tl.constexpr,
|
||||
IS_HIP: tl.constexpr,
|
||||
):
|
||||
# Keep token count dynamic to avoid compiling every prefill batch length.
|
||||
heads = tl.program_id(0) * BLOCK_HEADS + tl.arange(0, BLOCK_HEADS)
|
||||
@@ -56,7 +69,11 @@ def _q_rope_store_prefill(
|
||||
sin = tl.load(F + freq_offset + 1, rope_mask, 0)
|
||||
# Keep the same arithmetic and BF16 rounding as the decode kernel.
|
||||
even = tl.fma(value, cos, -partner * sin)
|
||||
odd = tl.fma(partner, sin, value * cos)
|
||||
if IS_HIP:
|
||||
# Match AMD flat RoPE: round the sine product before the cosine FMA.
|
||||
odd = tl.fma(value, cos, partner * sin)
|
||||
else:
|
||||
odd = tl.fma(partner, sin, value * cos)
|
||||
rotated = tl.where((r[None, :] & 1) == 0, even, odd)
|
||||
y_offset = row[:, None].to(tl.int64) * SY + head[:, None] * 512 + r[None, :]
|
||||
tl.store(
|
||||
@@ -91,6 +108,7 @@ def q_rope_store(
|
||||
q.stride(0),
|
||||
output.stride(0),
|
||||
BLOCK_HEADS=4,
|
||||
IS_HIP=is_hip(),
|
||||
num_warps=4,
|
||||
)
|
||||
return
|
||||
@@ -102,6 +120,7 @@ def q_rope_store(
|
||||
positions,
|
||||
q.stride(0),
|
||||
output.stride(0),
|
||||
IS_HIP=is_hip(),
|
||||
num_warps=4,
|
||||
**pdl_kwargs,
|
||||
)
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
"""Native 16-head MFMA layouts for gfx950 paged attention."""
|
||||
|
||||
import functools
|
||||
|
||||
from triton.experimental import gluon
|
||||
from triton.experimental.gluon import language as gl
|
||||
|
||||
from .decode_attention_sm100 import swapab_attention as _swapab_attention
|
||||
|
||||
# the V4 page load is plain gluon (no Blackwell ops), so the SM100 kernel's serves gfx950 too
|
||||
from .decode_attention_sm100_gluon import _load_v4
|
||||
|
||||
|
||||
@gluon.jit
|
||||
def partial_gluon(
|
||||
Q,
|
||||
K,
|
||||
E,
|
||||
IDX,
|
||||
EI,
|
||||
L,
|
||||
EL,
|
||||
PART,
|
||||
MAX,
|
||||
SUM,
|
||||
QS: gl.constexpr,
|
||||
QH: gl.constexpr,
|
||||
IS: gl.constexpr,
|
||||
EIS: gl.constexpr,
|
||||
KP: gl.constexpr,
|
||||
KS: gl.constexpr,
|
||||
EP: gl.constexpr,
|
||||
ES: gl.constexpr,
|
||||
NK: gl.constexpr,
|
||||
NE: gl.constexpr,
|
||||
NT: gl.constexpr,
|
||||
KT: gl.constexpr,
|
||||
BT: gl.constexpr,
|
||||
H: gl.constexpr,
|
||||
SCALE: gl.constexpr,
|
||||
KTOKENS: gl.constexpr,
|
||||
ETOKENS: gl.constexpr,
|
||||
COMPENSATE: gl.constexpr,
|
||||
SWAP_AB: gl.constexpr,
|
||||
DATA_BYTES: gl.constexpr,
|
||||
SCALE_BYTES: gl.constexpr,
|
||||
TILE: gl.constexpr,
|
||||
):
|
||||
gl.static_assert(SWAP_AB and H == 16 and (BT == 64 or BT == 128))
|
||||
b, t = gl.program_id(0), gl.program_id(1)
|
||||
kv_layout: gl.constexpr = gl.BlockedLayout([1, 8], [8, 8], [1, 4], [1, 0])
|
||||
n = gl.arange(0, BT, gl.SliceLayout(1, kv_layout))
|
||||
if t < KT:
|
||||
at = t * BT + n
|
||||
length = gl.load(L + b)
|
||||
ids = gl.load(IDX + b * IS + at, at < NK, -1)
|
||||
valid = (at < NK) & (at < length) & (ids >= 0) & (ids < KTOKENS)
|
||||
kv = _load_v4(
|
||||
K,
|
||||
gl.maximum(ids, 0),
|
||||
valid,
|
||||
KP,
|
||||
KS,
|
||||
kv_layout,
|
||||
DATA_BYTES,
|
||||
SCALE_BYTES,
|
||||
TILE,
|
||||
)
|
||||
else:
|
||||
at = (t - KT) * BT + n
|
||||
length = gl.load(EL + b)
|
||||
ids = gl.load(EI + b * EIS + at, at < NE, -1)
|
||||
valid = (at < NE) & (at < length) & (ids >= 0) & (ids < ETOKENS)
|
||||
kv = _load_v4(
|
||||
E,
|
||||
gl.maximum(ids, 0),
|
||||
valid,
|
||||
EP,
|
||||
ES,
|
||||
kv_layout,
|
||||
DATA_BYTES,
|
||||
SCALE_BYTES,
|
||||
TILE,
|
||||
)
|
||||
qh = gl.arange(0, H, gl.SliceLayout(1, kv_layout))
|
||||
qd = gl.arange(0, 512, gl.SliceLayout(0, kv_layout))
|
||||
q = gl.load(Q + b * QS + qh[:, None] * QH + qd[None, :])
|
||||
mf: gl.constexpr = gl.amd.AMDMFMALayout(
|
||||
version=4, instr_shape=[16, 16, 16], transposed=True, warps_per_cta=[4, 1]
|
||||
)
|
||||
scores = (
|
||||
gl.amd.cdna4.mfma(
|
||||
gl.convert_layout(kv, gl.DotOperandLayout(0, mf, 8)),
|
||||
gl.convert_layout(q.permute((1, 0)), gl.DotOperandLayout(1, mf, 8)),
|
||||
gl.zeros((BT, H), gl.float32, mf),
|
||||
)
|
||||
* SCALE
|
||||
)
|
||||
valid = gl.convert_layout(valid, gl.SliceLayout(1, mf))
|
||||
scores = gl.where(valid[:, None], scores, -float("inf"))
|
||||
mx = gl.max(scores, 0)
|
||||
mx = gl.where(mx == -float("inf"), 0.0, mx)
|
||||
prob = gl.exp(scores - mx[None, :])
|
||||
denom = gl.sum(prob, 0)
|
||||
hi = prob.to(gl.bfloat16)
|
||||
value = gl.amd.cdna4.mfma(
|
||||
gl.convert_layout(kv.permute((1, 0)), gl.DotOperandLayout(0, mf, 8)),
|
||||
gl.convert_layout(hi, gl.DotOperandLayout(1, mf, 8)),
|
||||
gl.zeros((512, H), gl.float32, mf),
|
||||
)
|
||||
if COMPENSATE:
|
||||
lo = (prob - hi.to(gl.float32)).to(gl.bfloat16)
|
||||
value = gl.amd.cdna4.mfma(
|
||||
gl.convert_layout(kv.permute((1, 0)), gl.DotOperandLayout(0, mf, 8)),
|
||||
gl.convert_layout(lo, gl.DotOperandLayout(1, mf, 8)),
|
||||
value,
|
||||
)
|
||||
hd: gl.constexpr = gl.BlockedLayout([1, 8], [8, 8], [1, 4], [1, 0])
|
||||
value = gl.convert_layout(value.permute((1, 0)), hd)
|
||||
h = gl.arange(0, H, gl.SliceLayout(1, hd))
|
||||
d = gl.arange(0, 512, gl.SliceLayout(0, hd))
|
||||
gl.store(PART + ((b * NT + t) * H + h[:, None]) * 512 + d[None, :], value)
|
||||
stat: gl.constexpr = gl.BlockedLayout([1], [64], [4], [0])
|
||||
hs = gl.arange(0, H, stat)
|
||||
gl.store(MAX + (b * NT + t) * H + hs, gl.convert_layout(mx, stat))
|
||||
gl.store(SUM + (b * NT + t) * H + hs, gl.convert_layout(denom, stat))
|
||||
|
||||
|
||||
# the SM100 host path (sink, combine, inverse RoPE) on the gfx950 split-KV kernel above
|
||||
swapab_attention = functools.partial(_swapab_attention, partial_gluon=partial_gluon)
|
||||
@@ -0,0 +1,133 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Split-KV combine for aiter's gfx950 sparse decode kernel, bitwise _pa_decode_sparse_reduce,
|
||||
with the DeepSeek-V4 inverse RoPE folded in."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Optional, Tuple
|
||||
|
||||
import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
from sglang.kernels.ops.attention.dsv4.decode_attention_sm100 import inverse_rope_tail
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _gfx_sparse_decode_reduce_kernel(
|
||||
part_m_ptr,
|
||||
part_l_ptr,
|
||||
part_acc_ptr,
|
||||
sink_ptr,
|
||||
out_ptr,
|
||||
fr_ptr,
|
||||
pos_ptr,
|
||||
pm_stride_t,
|
||||
pm_stride_s,
|
||||
pa_stride_t,
|
||||
pa_stride_s,
|
||||
pa_stride_h,
|
||||
out_stride_t,
|
||||
out_stride_h,
|
||||
fr_stride_pos,
|
||||
NUM_SPLITS: tl.constexpr,
|
||||
D: tl.constexpr,
|
||||
RD: tl.constexpr,
|
||||
HAS_SINK: tl.constexpr,
|
||||
INV_ROPE: tl.constexpr,
|
||||
):
|
||||
"""Grid (num_queries, num_heads). Partials: part_m / part_l
|
||||
[T, S, H] fp32 (m in the base-2 exponent domain), part_acc
|
||||
[T, S, H, D] fp32 -- the layout aiter's gluon main kernel stores."""
|
||||
RCP_LN2: tl.constexpr = 1.4426950408889634
|
||||
t = tl.program_id(0)
|
||||
h = tl.program_id(1)
|
||||
d = tl.arange(0, D)
|
||||
pm_base = t * pm_stride_t + h
|
||||
pa_base = t * pa_stride_t + h * pa_stride_h
|
||||
|
||||
neg_inf = float("-inf")
|
||||
m_final = tl.full((), neg_inf, tl.float32)
|
||||
for s in tl.static_range(NUM_SPLITS):
|
||||
m_s = tl.load(part_m_ptr + pm_base + s * pm_stride_s)
|
||||
m_final = tl.maximum(m_final, m_s)
|
||||
if HAS_SINK:
|
||||
sink = tl.load(sink_ptr + h).to(tl.float32)
|
||||
m_final = tl.maximum(m_final, sink * RCP_LN2)
|
||||
|
||||
l_final = tl.full((), 0.0, tl.float32)
|
||||
acc = tl.zeros([D], dtype=tl.float32)
|
||||
for s in tl.static_range(NUM_SPLITS):
|
||||
m_s = tl.load(part_m_ptr + pm_base + s * pm_stride_s)
|
||||
l_s = tl.load(part_l_ptr + pm_base + s * pm_stride_s)
|
||||
w = tl.exp2(m_s - m_final)
|
||||
# gluon contracts l + w * l_s into one fma; Triton leaves the scalar chain as mul + add
|
||||
l_final = tl.fma(w, l_s, l_final)
|
||||
acc_s = tl.load(part_acc_ptr + pa_base + s * pa_stride_s + d)
|
||||
acc = tl.fma(w, acc_s, acc)
|
||||
if HAS_SINK:
|
||||
l_final = l_final + tl.exp2(sink * RCP_LN2 - m_final)
|
||||
|
||||
out = (acc / l_final).to(out_ptr.dtype.element_ty)
|
||||
if INV_ROPE:
|
||||
pos = tl.load(pos_ptr + t)
|
||||
out = inverse_rope_tail(out, d, fr_ptr, pos, fr_stride_pos, D, RD)
|
||||
tl.store(out_ptr + t * out_stride_t + h * out_stride_h + d, out)
|
||||
|
||||
|
||||
def gfx_sparse_split_reduce(
|
||||
part_acc: torch.Tensor,
|
||||
part_m: torch.Tensor,
|
||||
part_l: torch.Tensor,
|
||||
attn_sink: Optional[torch.Tensor],
|
||||
out_dtype: torch.dtype = torch.bfloat16,
|
||||
inv_rope: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
|
||||
) -> torch.Tensor:
|
||||
"""Combine pa_decode_sparse(..., skip_reduce=True)'s partials into the
|
||||
[T, H, D] output. inv_rope is (freqs_real [max_pos, rope_dim] fp32,
|
||||
positions [T]): the last rope_dim of every head are inverse-rotated."""
|
||||
T, S, H, D = part_acc.shape
|
||||
assert part_m.shape == (T, S, H) and part_l.shape == (T, S, H), (
|
||||
part_m.shape,
|
||||
part_l.shape,
|
||||
)
|
||||
assert part_acc.stride(3) == 1 and part_m.stride(2) == 1 and part_l.stride(2) == 1
|
||||
assert part_m.stride() == part_l.stride()
|
||||
assert triton.next_power_of_2(D) == D and S >= 1
|
||||
out = torch.empty((T, H, D), dtype=out_dtype, device=part_acc.device)
|
||||
if T == 0:
|
||||
return out
|
||||
has_sink = attn_sink is not None
|
||||
if inv_rope is not None:
|
||||
freqs_real, positions = inv_rope
|
||||
rope_dim = freqs_real.shape[1]
|
||||
assert freqs_real.dtype == torch.float32 and freqs_real.stride(1) == 1
|
||||
assert positions.shape == (T,), positions.shape
|
||||
assert rope_dim % 2 == 0 and rope_dim <= D
|
||||
else:
|
||||
freqs_real = positions = out # unread placeholders
|
||||
rope_dim = 0
|
||||
_gfx_sparse_decode_reduce_kernel[(T, H)](
|
||||
part_m,
|
||||
part_l,
|
||||
part_acc,
|
||||
attn_sink if has_sink else out,
|
||||
out,
|
||||
freqs_real,
|
||||
positions,
|
||||
part_m.stride(0),
|
||||
part_m.stride(1),
|
||||
part_acc.stride(0),
|
||||
part_acc.stride(1),
|
||||
part_acc.stride(2),
|
||||
out.stride(0),
|
||||
out.stride(1),
|
||||
freqs_real.stride(0),
|
||||
NUM_SPLITS=S,
|
||||
D=D,
|
||||
RD=rope_dim,
|
||||
HAS_SINK=has_sink,
|
||||
INV_ROPE=inv_rope is not None,
|
||||
num_warps=1,
|
||||
)
|
||||
return out
|
||||
Reference in New Issue
Block a user