dsv4.1-amd: gfx950 sparse decode attention and sorted top-k (#41020)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Kevin Mi
2026-09-28 16:14:28 -07:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 26d7539799
commit 096b066fb4
17 changed files with 2569 additions and 17 deletions
@@ -0,0 +1,76 @@
"""Benchmark for the DeepSeek-V4 indexer top-k transform, with and without sorted output."""
import itertools
import torch
import triton
import triton.testing
from sgl_kernel import deepseek_v4_topk_transform_512
try:
from sglang.utils import is_in_ci
IS_CI = is_in_ci()
except ImportError:
IS_CI = False
TOPK = 512
PAGE_SIZE = 64
batch_sizes = [1] if IS_CI else [1, 8, 32, 128]
seq_lens = [4096] if IS_CI else [1024, 4096, 32768]
configs = list(itertools.product(batch_sizes, seq_lens))
@triton.testing.perf_report(
triton.testing.Benchmark(
x_names=["batch_size", "seq_len"],
x_vals=configs,
line_arg="provider",
line_vals=["unsorted", "sorted", "torch_sort"],
line_names=["SGL Kernel", "SGL Kernel sort_output", "SGL Kernel + torch.sort"],
styles=[("blue", "-"), ("green", "-"), ("red", "--")],
ylabel="µs (median)",
plot_name="dsv4-topk-transform-performance",
args={},
)
)
def benchmark_topk_transform(batch_size, seq_len, provider):
torch.manual_seed(42)
scores = torch.randn(batch_size, seq_len, dtype=torch.float32, device="cuda")
lengths = torch.full((batch_size,), seq_len, dtype=torch.int32, device="cuda")
num_pages = triton.cdiv(seq_len, PAGE_SIZE)
page_table = torch.stack(
[
torch.randperm(4 * num_pages, device="cuda")[:num_pages]
for _ in range(batch_size)
]
).int()
page_indices = torch.empty(batch_size, TOPK, dtype=torch.int32, device="cuda")
def topk(sort_output):
deepseek_v4_topk_transform_512(
scores,
lengths,
page_table,
page_indices,
PAGE_SIZE,
sort_output=sort_output,
)
if provider == "unsorted":
fn = lambda: topk(False)
elif provider == "sorted":
fn = lambda: topk(True)
else:
fn = lambda: (topk(False), torch.sort(page_indices, dim=-1))
ms, min_ms, max_ms = triton.testing.do_bench_cudagraph(
fn, quantiles=[0.5, 0.2, 0.8]
)
return 1000 * ms, 1000 * max_ms, 1000 * min_ms
if __name__ == "__main__":
benchmark_topk_transform.run(print_data=True)
@@ -49,7 +49,7 @@ TORCH_LIBRARY_EXPAND(sgl_kernel, m) {
m.def(
"deepseek_v4_topk_transform_512(Tensor scores, Tensor seq_lens, Tensor page_table, Tensor! "
"page_indices, int page_size, Tensor!? raw_indices) -> ()");
"page_indices, int page_size, Tensor!? raw_indices, bool sort_output=False) -> ()");
m.impl("deepseek_v4_topk_transform_512", torch::kCUDA, &deepseek_v4_topk_transform_512);
m.def(
@@ -46,6 +46,7 @@ constexpr size_t kSMEM = 48 * 1024; // bytes
#endif
static_assert(kSMEM % (2 * sizeof(int32_t)) == 0, "kSMEM must be a multiple of 8 bytes.");
// seq_lens[b] must not exceed scores.size(1) or page_table.size(1) << page_bits: a row reads up to its length
struct TopKParams {
const float* __restrict__ scores;
const int32_t* __restrict__ seq_lens;
@@ -57,6 +58,8 @@ struct TopKParams {
uint32_t page_bits;
uint32_t topk;
int64_t output_stride;
// Emit each row's picks in ascending order (see bitonic_sort_u32).
bool sort_output;
};
__device__ __forceinline__ uint8_t convert_to_uint8(float x) {
@@ -251,6 +254,116 @@ radix_topk(const float* __restrict__ input, int32_t* __restrict__ output, uint32
}
}
#ifdef USE_ROCM
// Bitonic sort of n (a power of two, 64 <= n <= kMaxTopK) 32-bit keys, one per thread: strides
// below the wavefront width exchange through lane shuffles, the wider ones through LDS.
#if defined(__GFX10__) || defined(__GFX11__) || defined(__GFX12__)
constexpr uint32_t kWaveSize = 32; // RDNA and gfx1250 run 32-lane waves
#else
constexpr uint32_t kWaveSize = 64;
#endif
// lane ^ J's value in registers: DPP for J <= 8, gfx950 permlane swaps for J = 16, 32; __shfl_xor
// is an LDS round trip on every stage's dependent chain.
template <uint32_t J>
__device__ __forceinline__ uint32_t lane_xor(uint32_t v) {
#if defined(__gfx90a__) || defined(__gfx942__) || defined(__gfx950__)
if constexpr (J == 1) { // quad_perm [1, 0, 3, 2]
return static_cast<uint32_t>(__builtin_amdgcn_update_dpp(0, static_cast<int>(v), 0xB1, 0xF, 0xF, true));
}
if constexpr (J == 2) { // quad_perm [2, 3, 0, 1]
return static_cast<uint32_t>(__builtin_amdgcn_update_dpp(0, static_cast<int>(v), 0x4E, 0xF, 0xF, true));
}
if constexpr (J == 4 || J == 8) {
// within a 16-lane row: bit clear reads J lanes up (row_shl), bit set J lanes down (row_shr)
const uint32_t shl =
static_cast<uint32_t>(__builtin_amdgcn_update_dpp(0, static_cast<int>(v), 0x100 | J, 0xF, 0xF, true));
const uint32_t shr =
static_cast<uint32_t>(__builtin_amdgcn_update_dpp(0, static_cast<int>(v), 0x110 | J, 0xF, 0xF, true));
return (__lane_id() & J) ? shr : shl;
}
#endif
#if defined(__gfx950__)
if constexpr (J == 16) {
// rows 1 and 3 of the first operand swap with rows 0 and 2 of the second
const auto pair = __builtin_amdgcn_permlane16_swap(v, v, false, false);
return (__lane_id() & 16) ? pair[0] : pair[1];
}
if constexpr (J == 32) {
const auto pair = __builtin_amdgcn_permlane32_swap(v, v, false, false);
return (__lane_id() & 32) ? pair[0] : pair[1];
}
#endif
return static_cast<uint32_t>(__shfl_xor(static_cast<int>(v), static_cast<int>(J), kWaveSize));
}
// One bitonic stage (merge size K, stride J); templated so the network unrolls with no stride switch.
template <uint32_t K, uint32_t J>
__device__ __forceinline__ uint32_t bitonic_stage(uint32_t v, uint32_t* __restrict__ s_vals, uint32_t tx, uint32_t n) {
const bool up = (tx & K) == 0;
const bool lower = (tx & J) == 0;
uint32_t w;
if constexpr (J >= kWaveSize) {
__syncthreads();
if (tx < n) s_vals[tx] = v;
__syncthreads();
w = tx < n ? s_vals[tx ^ J] : ~0u;
} else {
w = lane_xor<J>(v);
}
return (lower == up) ? min(v, w) : max(v, w);
}
template <uint32_t N, uint32_t K, uint32_t J>
__device__ __forceinline__ uint32_t bitonic_network(uint32_t v, uint32_t* __restrict__ s_vals, uint32_t tx) {
v = bitonic_stage<K, J>(v, s_vals, tx, N);
if constexpr (J > 1) {
return bitonic_network<N, K, J / 2>(v, s_vals, tx);
} else if constexpr (K < N) {
return bitonic_network<N, K * 2, K>(v, s_vals, tx);
} else {
return v;
}
}
// Sort the N (a power of two, 64 <= N <= kBlockSize) values in s_vals ascending, one per thread.
template <uint32_t N>
__device__ void bitonic_sort_fixed(uint32_t* __restrict__ s_vals) {
static_assert(N >= 64 && N <= kBlockSize && (N & (N - 1)) == 0, "one value per thread");
const uint32_t tx = threadIdx.x;
uint32_t v = tx < N ? s_vals[tx] : ~0u;
v = bitonic_network<N, 2, 1>(v, s_vals, tx);
__syncthreads();
if (tx < N) s_vals[tx] = v;
__syncthreads();
}
// one value per thread: the block covers the largest top-k
static_assert(kBlockSize >= kMaxTopK);
__device__ void bitonic_sort_u32(uint32_t* __restrict__ s_vals, uint32_t n) {
switch (n) {
case 64:
return bitonic_sort_fixed<64>(s_vals);
case 128:
return bitonic_sort_fixed<128>(s_vals);
case 256:
return bitonic_sort_fixed<256>(s_vals);
case 512:
return bitonic_sort_fixed<512>(s_vals);
default:
return bitonic_sort_fixed<kMaxTopK>(s_vals);
}
}
__device__ __forceinline__ uint32_t next_pow2_at_least_64(uint32_t x) {
uint32_t n = 64;
while (n < x)
n <<= 1;
return n;
}
#endif // USE_ROCM
__global__ __launch_bounds__(kBlockSize) void deepseek_v4_topk_transform_kernel(const TopKParams params) {
const auto bid = blockIdx.x;
const auto seq_len = params.seq_lens[bid];
@@ -261,15 +374,63 @@ __global__ __launch_bounds__(kBlockSize) void deepseek_v4_topk_transform_kernel(
const auto raw_indices_ptr =
params.raw_indices != nullptr ? params.raw_indices + bid * params.output_stride : nullptr;
__shared__ int32_t s_topk_indices[kMaxTopK];
// key: the position when the row has raw indices, else the slot
const bool key_is_position = raw_indices_ptr != nullptr;
uint32_t count = topk;
if (seq_len <= static_cast<int32_t>(topk)) {
naive_paged_transform(seq_len, topk, params.page_bits, page_ptr, indices_ptr, raw_indices_ptr);
if (!params.sort_output || key_is_position) {
// ascending positions with the -1 padding last: already the sorted row
naive_paged_transform(seq_len, topk, params.page_bits, page_ptr, indices_ptr, raw_indices_ptr);
return;
}
// every position is a pick; the slot order still has to be established
count = static_cast<uint32_t>(seq_len);
for (uint32_t i = threadIdx.x; i < count; i += kBlockSize) {
s_topk_indices[i] = static_cast<int32_t>(i);
}
} else {
radix_topk(score_ptr, s_topk_indices, static_cast<uint32_t>(seq_len), topk);
}
__syncthreads();
#ifdef USE_ROCM
if (params.sort_output) {
__shared__ uint32_t s_sort_vals[kMaxTopK];
const uint32_t n = next_pow2_at_least_64(count);
for (uint32_t i = threadIdx.x; i < n; i += kBlockSize) {
uint32_t key = ~0u;
if (i < count) {
const int32_t raw = s_topk_indices[i];
key = static_cast<uint32_t>(
key_is_position ? raw : page_to_slot(page_ptr, static_cast<uint32_t>(raw), params.page_bits));
}
s_sort_vals[i] = key;
}
__syncthreads();
bitonic_sort_u32(s_sort_vals, n);
for (uint32_t i = threadIdx.x; i < topk; i += kBlockSize) {
int32_t slot = -1;
int32_t raw = -1;
if (i < count) {
const int32_t key = static_cast<int32_t>(s_sort_vals[i]);
if (key_is_position) {
raw = key;
slot = page_to_slot(page_ptr, static_cast<uint32_t>(raw), params.page_bits);
} else {
slot = key;
}
}
indices_ptr[i] = slot;
if (raw_indices_ptr != nullptr) {
raw_indices_ptr[i] = raw;
}
}
return;
}
#endif
__shared__ int32_t s_topk_indices[kMaxTopK];
radix_topk(score_ptr, s_topk_indices, static_cast<uint32_t>(seq_len), topk);
__syncthreads();
for (uint32_t i = threadIdx.x; i < topk; i += kBlockSize) {
const auto raw = s_topk_indices[i];
indices_ptr[i] = page_to_slot(page_ptr, static_cast<uint32_t>(raw), params.page_bits);
@@ -304,11 +465,15 @@ void deepseek_v4_topk_transform_512(
const at::Tensor& page_table,
at::Tensor& page_indices,
int64_t page_size,
std::optional<at::Tensor> raw_indices_opt) {
std::optional<at::Tensor> raw_indices_opt,
bool sort_output) {
CHECK_CUDA(scores);
CHECK_CUDA(seq_lens);
CHECK_CUDA(page_table);
CHECK_CUDA(page_indices);
#ifndef USE_ROCM
TORCH_CHECK(!sort_output, "deepseek_v4_topk_transform_512: sort_output is only built for ROCm");
#endif
if (raw_indices_opt.has_value()) {
CHECK_CUDA(raw_indices_opt.value());
}
@@ -367,6 +532,7 @@ void deepseek_v4_topk_transform_512(
.page_bits = page_bits,
.topk = static_cast<uint32_t>(topk),
.output_stride = topk,
.sort_output = sort_output,
};
const auto stream = at::cuda::getCurrentCUDAStream().stream();
@@ -170,7 +170,8 @@ void deepseek_v4_topk_transform_512(
const at::Tensor& page_table,
at::Tensor& page_indices,
int64_t page_size,
std::optional<at::Tensor> raw_indices_opt = std::nullopt);
std::optional<at::Tensor> raw_indices_opt = std::nullopt,
bool sort_output = false);
#endif
/*
@@ -87,6 +87,7 @@ def deepseek_v4_topk_transform_512(
page_indices: torch.Tensor,
page_size: int,
raw_indices: Optional[torch.Tensor] = None,
sort_output: bool = False,
) -> None:
"""
Performs the DeepSeek-V4 indexer top-k selection and writes the paged
@@ -96,7 +97,8 @@ def deepseek_v4_topk_transform_512(
Args:
scores: float32 ``[B, max_seq_len]`` indexer logits, contiguous on dim 1.
seq_lens: int32 ``[B]``, true KV length per batch row.
seq_lens: int32 ``[B]``, true KV length per batch row; each at most
``max_seq_len`` and ``num_pages * page_size``.
page_table: int32 ``[B, num_pages]``, logical->physical page table,
contiguous on dim 1.
page_indices: int32 ``[B, topk]``, output buffer, contiguous. Filled
@@ -104,11 +106,14 @@ def deepseek_v4_topk_transform_512(
page_size: power-of-2 page size.
raw_indices: optional int32 ``[B, topk]``, contiguous. If provided,
filled with raw token positions within each row.
sort_output: sort every row ascending in the kernel epilogue, with the
-1 padding last: by raw position when ``raw_indices`` is given,
otherwise by physical slot. The selected set is unchanged.
"""
if raw_indices is not None:
assert raw_indices.dim() == 2
torch.ops.sgl_kernel.deepseek_v4_topk_transform_512(
scores, seq_lens, page_table, page_indices, page_size, raw_indices
scores, seq_lens, page_table, page_indices, page_size, raw_indices, sort_output
)
@@ -292,6 +292,51 @@ def test_deepseek_v4_topk_transform(bs: int, c4_len: int) -> None:
)
@pytest.mark.skipif(
torch.version.hip is None,
reason="deepseek_v4_topk_transform_512 is only built on ROCm",
)
@pytest.mark.parametrize("with_raw", [False, True])
@torch.inference_mode()
def test_deepseek_v4_topk_transform_sorted_output(with_raw: bool) -> None:
"""sort_output orders each row ascending (by raw position when given), -1 padding last,
without changing the selected set, also for rows shorter than topk."""
from sgl_kernel import deepseek_v4_topk_transform_512
torch.manual_seed(7)
topk, page_size, width = 512, 64, 5000
lens = [0, 1, topk - 1, topk, topk + 1, 3000, width]
bs = len(lens)
scores = torch.randn(bs, width, dtype=torch.float32, device="cuda")
seq_lens = torch.tensor(lens, dtype=torch.int32, device="cuda")
num_pages = (width + page_size - 1) // page_size
page_table = torch.stack(
[torch.randperm(4 * num_pages, device="cuda")[:num_pages] for _ in range(bs)]
).int()
outs = []
for sort_output in (False, True):
page = torch.empty(bs, topk, dtype=torch.int32, device="cuda")
raw = torch.empty_like(page) if with_raw else None
deepseek_v4_topk_transform_512(
scores, seq_lens, page_table, page, page_size, raw, sort_output=sort_output
)
outs.append((page, raw))
(page, raw), (page_sorted, raw_sorted) = outs
for b in range(bs):
key = (raw if with_raw else page)[b]
chosen = key[key >= 0].sort().values
padding = torch.full(
(topk - chosen.numel(),), -1, dtype=torch.int32, device="cuda"
)
assert torch.equal(
(raw_sorted if with_raw else page_sorted)[b], torch.cat([chosen, padding])
)
if with_raw: # each slot still travels with its raw position
pairs = sorted(zip(page[b].tolist(), raw[b].tolist()))
assert pairs == sorted(zip(page_sorted[b].tolist(), raw_sorted[b].tolist()))
def _make_scores(kind: str, bs: int, width: int, seed: int) -> torch.Tensor:
"""Score distributions that stress the coarse stage of a histogram top-k.
@@ -0,0 +1,426 @@
"""Single-launch replacements for the torch glue around the DeepSeek-V4.1 sparse attention metadata
on HIP; every kernel reproduces the torch expression quoted in its docstring bit for bit."""
from __future__ import annotations
from typing import Dict, Optional, Tuple
import torch
import triton
import triton.language as tl
@triton.jit
def _mask_indices_by_length_kernel(
idx_ptr,
len_ptr,
out_ptr,
idx2_ptr,
len2_ptr,
out2_ptr,
rows_per_len,
W1: tl.constexpr,
W2: tl.constexpr,
BLOCK: tl.constexpr,
HAS_SECOND: tl.constexpr,
):
row = tl.program_id(0)
which = tl.program_id(1)
length_row = row // rows_per_len
offs = tl.arange(0, BLOCK)
if which == 0:
length = tl.load(len_ptr + length_row)
for w0 in tl.static_range(0, W1, BLOCK):
w = w0 + offs
m = w < W1
v = tl.load(idx_ptr + row * W1 + w, mask=m, other=0)
tl.store(out_ptr + row * W1 + w, tl.where(w < length, v, -1), mask=m)
elif HAS_SECOND:
length = tl.load(len2_ptr + length_row)
for w0 in tl.static_range(0, W2, BLOCK):
w = w0 + offs
m = w < W2
v = tl.load(idx2_ptr + row * W2 + w, mask=m, other=0)
tl.store(out2_ptr + row * W2 + w, tl.where(w < length, v, -1), mask=m)
def _mask_shape(indices: torch.Tensor, lengths: torch.Tensor) -> Tuple[int, int, int]:
assert indices.is_contiguous(), indices.stride()
assert lengths.dim() == 1 and lengths.is_contiguous()
w = indices.shape[-1]
rows = indices.numel() // w if w else 0
batch = lengths.shape[0]
assert batch > 0 and rows % batch == 0, (indices.shape, lengths.shape)
return rows, rows // batch, w
def mask_indices_by_length(
indices: torch.Tensor,
lengths: torch.Tensor,
indices2: Optional[torch.Tensor] = None,
lengths2: Optional[torch.Tensor] = None,
) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
"""torch.where(arange(w) < lengths.view(-1, 1, 1), indices, -1) for one or
two [b, s, w] index tensors in one launch (the sparse decode kernel takes
no per-row length and skips -1). lengths is [b]; every s row of a
batch entry shares its length. Returns new tensors of the input dtypes."""
rows, rows_per_len, w1 = _mask_shape(indices, lengths)
out = torch.empty_like(indices)
has_second = indices2 is not None
if has_second:
rows2, rows_per_len2, w2 = _mask_shape(indices2, lengths2)
assert rows2 == rows and rows_per_len2 == rows_per_len, (
indices.shape,
indices2.shape,
)
out2 = torch.empty_like(indices2)
else:
w2 = w1
indices2, lengths2, out2 = indices, lengths, out
if rows == 0:
return out, (out2 if has_second else None)
block = min(1024, triton.next_power_of_2(max(w1, w2)))
_mask_indices_by_length_kernel[(rows, 2 if has_second else 1)](
indices,
lengths,
out,
indices2,
lengths2,
out2,
rows_per_len,
W1=w1,
W2=w2,
BLOCK=block,
HAS_SECOND=has_second,
num_warps=4,
)
return out, (out2 if has_second else None)
@triton.jit
def _expand_index_page_table_kernel(
pt_ptr,
out_ptr,
n_pages,
pt_stride,
BPP: tl.constexpr,
BLOCK: tl.constexpr,
):
row = tl.program_id(0)
p = tl.program_id(1) * BLOCK + tl.arange(0, BLOCK)
m = p < n_pages
page = tl.load(pt_ptr + row * pt_stride + p, mask=m, other=0)
base = page.to(tl.int64) * BPP
k = tl.arange(0, BPP)
expanded = base[:, None] + k[None, :]
tl.store(
out_ptr + row * (n_pages * BPP) + p[:, None] * BPP + k[None, :],
expanded.to(tl.int32),
mask=m[:, None],
)
def expand_index_page_table(
page_table: torch.Tensor, blocks_per_page: int
) -> torch.Tensor:
"""(page_table.to(int64) * bpp).unsqueeze(-1) + arange(bpp) reshaped to
[bs, n * bpp] int32: the block table of a low-ratio indexer-K pool that
pages at a fraction of a FULL page (candidate_indexer.expand_index_page_table)."""
if blocks_per_page == 1:
return page_table
assert blocks_per_page & (blocks_per_page - 1) == 0, blocks_per_page
assert page_table.dim() == 2 and page_table.stride(1) == 1
bs, n = page_table.shape
out = torch.empty(
(bs, n * blocks_per_page), dtype=torch.int32, device=page_table.device
)
if bs == 0 or n == 0:
return out
block = 256
_expand_index_page_table_kernel[(bs, triton.cdiv(n, block))](
page_table,
out,
n,
page_table.stride(0),
BPP=blocks_per_page,
BLOCK=block,
num_warps=4,
)
return out
@triton.jit
def _low_ratio_compression_metadata_kernel(
seq_lens_ptr,
raw_out_loc_ptr,
c1_out_loc_ptr,
c1_clamp1_ptr,
c2_out_loc_ptr,
c2_clamp1_ptr,
rows,
num_write_tokens,
HAS_C1: tl.constexpr,
HAS_C2: tl.constexpr,
BLOCK: tl.constexpr,
):
r = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)
m = r < rows
mw = r < num_write_tokens
seq_len = tl.load(seq_lens_ptr + r, mask=m, other=0)
raw_out_loc = tl.load(raw_out_loc_ptr + r, mask=mw, other=0).to(tl.int64)
if HAS_C1:
# ratio 1: every token completes a group, so out_loc is raw_out_loc itself
tl.store(c1_out_loc_ptr + r, raw_out_loc, mask=mw)
tl.store(c1_clamp1_ptr + r, tl.maximum(seq_len, 1), mask=m)
if HAS_C2:
completes = (seq_len % 2) == 0
# >> 1 floors as torch's // does; Triton's // rounds toward zero on a negative loc
tl.store(c2_out_loc_ptr + r, tl.where(completes, raw_out_loc >> 1, -1), mask=mw)
tl.store(c2_clamp1_ptr + r, tl.maximum(seq_len // 2, 1), mask=m)
def low_ratio_compression_metadata(
seq_lens_casual: torch.Tensor,
raw_out_loc: torch.Tensor,
low_ratios: Tuple[int, ...],
) -> Dict[str, torch.Tensor]:
"""_low_ratio_compression_metadata for ratios 1 and 2 in one launch: cR_out_loc =
where(seq_lens[:nw] % R == 0, raw_out_loc // R, -1) (int64) and cR_topk_lengths_clamp1 =
(seq_lens // R).clamp_min(1) (int32), keyed c{R}_... for the ratios present."""
assert seq_lens_casual.dtype is torch.int32 and seq_lens_casual.is_contiguous()
assert raw_out_loc.dim() == 1 and raw_out_loc.is_contiguous()
assert set(low_ratios) <= {1, 2}, low_ratios
rows = seq_lens_casual.shape[0]
nw = raw_out_loc.shape[0]
assert nw <= rows, (nw, rows)
device = seq_lens_casual.device
out = {}
for ratio in (1, 2):
if ratio in low_ratios:
out[f"c{ratio}_out_loc"] = torch.empty(nw, dtype=torch.int64, device=device)
out[f"c{ratio}_topk_lengths_clamp1"] = torch.empty(
rows, dtype=torch.int32, device=device
)
if not out or rows == 0:
return out
placeholder = seq_lens_casual
_low_ratio_compression_metadata_kernel[(triton.cdiv(rows, 1024),)](
seq_lens_casual,
raw_out_loc,
out.get("c1_out_loc", placeholder),
out.get("c1_topk_lengths_clamp1", placeholder),
out.get("c2_out_loc", placeholder),
out.get("c2_topk_lengths_clamp1", placeholder),
rows,
nw,
HAS_C1=1 in low_ratios,
HAS_C2=2 in low_ratios,
BLOCK=1024,
num_warps=4,
)
return out
@triton.jit
def _sparse_buffers_kernel(
c4_clamp1_ptr,
c4_raw_ptr,
c4_sparse_ptr,
c4_sparse_raw_ptr,
c4_page_ptr,
c1_clamp1_ptr,
c1_sparse_ptr,
c1_page_ptr,
c2_clamp1_ptr,
c2_sparse_ptr,
c2_page_ptr,
rows,
topk,
width,
HAS_C1: tl.constexpr,
HAS_C2: tl.constexpr,
BLOCK: tl.constexpr,
FILL_BLOCK: tl.constexpr,
):
pid = tl.program_id(0)
if pid == 0:
# one program clamps the per-row lengths: a few ints per row
for r0 in range(0, rows, BLOCK):
r = r0 + tl.arange(0, BLOCK)
m = r < rows
c4_clamp1 = tl.load(c4_clamp1_ptr + r, mask=m, other=0)
c4_raw = tl.load(c4_raw_ptr + r, mask=m, other=0)
tl.store(c4_sparse_ptr + r, tl.minimum(c4_clamp1, topk), mask=m)
tl.store(c4_sparse_raw_ptr + r, tl.minimum(c4_raw, topk), mask=m)
if HAS_C1:
c1_clamp1 = tl.load(c1_clamp1_ptr + r, mask=m, other=0)
tl.store(c1_sparse_ptr + r, tl.minimum(c1_clamp1, topk), mask=m)
if HAS_C2:
c2_clamp1 = tl.load(c2_clamp1_ptr + r, mask=m, other=0)
tl.store(c2_sparse_ptr + r, tl.minimum(c2_clamp1, topk), mask=m)
else:
# the rest fill the -1 page-index buffers, one buffer per range of programs
fill = pid - 1
n = rows * width
per_buffer = tl.cdiv(n, FILL_BLOCK)
which = fill // per_buffer
i = (fill - which * per_buffer) * FILL_BLOCK + tl.arange(0, FILL_BLOCK)
m = i < n
if which == 0:
tl.store(c4_page_ptr + i, -1, mask=m)
elif which == 1:
if HAS_C1:
tl.store(c1_page_ptr + i, -1, mask=m)
elif HAS_C2:
tl.store(c2_page_ptr + i, -1, mask=m)
else:
if HAS_C1 and HAS_C2:
tl.store(c2_page_ptr + i, -1, mask=m)
def sparse_buffers(
*,
c4_topk_lengths_clamp1: torch.Tensor,
c4_topk_lengths_raw: torch.Tensor,
c1_topk_lengths_clamp1: Optional[torch.Tensor],
c2_topk_lengths_clamp1: Optional[torch.Tensor],
index_topk: int,
page_index_align: int,
) -> Dict[str, torch.Tensor]:
"""init_flashmla_related's tensors in one launch, keyed by attribute name: the top-k
lengths clamped to index_topk and the -1-filled int32 page-index buffers padded to
page_index_align, for ratio 4 and the ratios whose clamp-1 lengths are given."""
for t in (
c4_topk_lengths_clamp1,
c4_topk_lengths_raw,
c1_topk_lengths_clamp1,
c2_topk_lengths_clamp1,
):
assert t is None or (
t.dtype is torch.int32 and t.dim() == 1 and t.is_contiguous()
), t
rows = c4_topk_lengths_clamp1.shape[0]
assert c4_topk_lengths_raw.shape[0] == rows
device = c4_topk_lengths_clamp1.device
width = triton.cdiv(index_topk, page_index_align) * page_index_align
i32 = dict(dtype=torch.int32, device=device)
out = {
"c4_sparse_topk_lengths": torch.empty(rows, **i32),
"c4_sparse_topk_lengths_raw": torch.empty(rows, **i32),
"c4_sparse_page_indices": torch.empty((rows, width), **i32),
}
has_c1 = c1_topk_lengths_clamp1 is not None
has_c2 = c2_topk_lengths_clamp1 is not None
for ratio, has in ((1, has_c1), (2, has_c2)):
if has:
out[f"c{ratio}_sparse_topk_lengths"] = torch.empty(rows, **i32)
out[f"c{ratio}_sparse_page_indices"] = torch.empty((rows, width), **i32)
if rows == 0:
return out
placeholder = out["c4_sparse_topk_lengths"]
fill_block = 1024
per_buffer = triton.cdiv(rows * width, fill_block)
n_buffers = 1 + int(has_c1) + int(has_c2)
_sparse_buffers_kernel[(1 + per_buffer * n_buffers,)](
c4_topk_lengths_clamp1,
c4_topk_lengths_raw,
out["c4_sparse_topk_lengths"],
out["c4_sparse_topk_lengths_raw"],
out["c4_sparse_page_indices"],
c1_topk_lengths_clamp1 if has_c1 else placeholder,
out.get("c1_sparse_topk_lengths", placeholder),
out.get("c1_sparse_page_indices", placeholder),
c2_topk_lengths_clamp1 if has_c2 else placeholder,
out.get("c2_sparse_topk_lengths", placeholder),
out.get("c2_sparse_page_indices", placeholder),
rows,
index_topk,
width,
HAS_C1=has_c1,
HAS_C2=has_c2,
BLOCK=1024,
FILL_BLOCK=fill_block,
num_warps=4,
)
return out
@triton.jit
def _page_table_from_req_to_token_kernel(
r2t_ptr,
req_ptr,
out_ptr,
num_cols,
r2t_stride,
PAGE: tl.constexpr,
BLOCK: tl.constexpr,
):
row = tl.program_id(0)
blk = tl.program_id(1)
req = tl.load(req_ptr + row).to(tl.int64)
col = blk * BLOCK + tl.arange(0, BLOCK)
mask = col < num_cols
v = tl.load(
r2t_ptr + req * r2t_stride + col.to(tl.int64) * PAGE, mask=mask, other=0
)
# torch's floor division; the C-style quotient rounds toward zero
q = v // PAGE
q = tl.where((v < 0) & (v % PAGE != 0), q - 1, q)
tl.store(out_ptr + row.to(tl.int64) * num_cols + col, q.to(tl.int32), mask=mask)
def page_table_from_req_to_token(
req_to_token: torch.Tensor,
req_rows: torch.Tensor,
max_seq_len: int,
page_size: int,
) -> torch.Tensor:
"""(req_to_token[req_rows, :max_seq_len:page_size] // page_size).to(torch.int32) in one
launch (the gather, the floor division and the cast)."""
assert req_to_token.dim() == 2 and req_to_token.stride(1) == 1
assert req_rows.dim() == 1 and req_rows.stride(0) == 1
n = req_rows.shape[0]
num_cols = max(
0, (min(max_seq_len, req_to_token.shape[1]) + page_size - 1) // page_size
)
out = torch.empty((n, num_cols), dtype=torch.int32, device=req_to_token.device)
if n == 0 or num_cols == 0:
return out
block = min(1024, triton.next_power_of_2(num_cols))
_page_table_from_req_to_token_kernel[(n, triton.cdiv(num_cols, block))](
req_to_token,
req_rows,
out,
num_cols,
req_to_token.stride(0),
PAGE=page_size,
BLOCK=block,
num_warps=4,
)
return out
@triton.jit
def _widen_pair_kernel(a_ptr, b_ptr, oa_ptr, ob_ptr, n, BLOCK: tl.constexpr):
i = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)
mask = i < n
tl.store(oa_ptr + i, tl.load(a_ptr + i, mask=mask, other=0).to(tl.int64), mask=mask)
tl.store(ob_ptr + i, tl.load(b_ptr + i, mask=mask, other=0).to(tl.int64), mask=mask)
def widen_pair_i64(
a: torch.Tensor, b: torch.Tensor
) -> Tuple[torch.Tensor, torch.Tensor]:
"""(a.to(torch.int64), b.to(torch.int64)) for two same-length integer vectors in one
launch (the decode request slots and positions the low-ratio indexer takes widened)."""
assert a.dim() == b.dim() == 1 and a.shape == b.shape, (a.shape, b.shape)
assert a.stride(0) == 1 and b.stride(0) == 1
n = a.shape[0]
oa = torch.empty(n, dtype=torch.int64, device=a.device)
ob = torch.empty(n, dtype=torch.int64, device=a.device)
if n:
block = min(1024, triton.next_power_of_2(n))
_widen_pair_kernel[(triton.cdiv(n, block),)](
a, b, oa, ob, n, BLOCK=block, num_warps=4
)
return oa, ob
@@ -0,0 +1,408 @@
"""Candidate blocks of the DeepSeek V4.1 two-level indexer on ROCm: level-one block scores and
top blocks, the top-k within them, and the sorted paged top-k (the HIP side of candidate_blocks)."""
from __future__ import annotations
from typing import List, NamedTuple, Optional
import torch
import torch.nn.functional as F
import triton
import triton.language as tl
from sglang.kernels.ops.attention.dsv4 import topk_transform_paged
# the k the AOT fast_topk op (topk_hip.hip) is instantiated for; any other k takes torch.topk
_AOT_FAST_TOPK_K = 2048
def topk_transform_paged_sorted(
scores: torch.Tensor,
seq_lens: torch.Tensor,
page_table: torch.Tensor,
page_indices: torch.Tensor,
page_size: int,
raw_indices: Optional[torch.Tensor],
) -> None:
"""topk_transform_paged with every row sorted ascending by position in the AOT
kernel's epilogue, -1 padding last."""
torch.ops.sgl_kernel.deepseek_v4_topk_transform_512(
scores, seq_lens, page_table, page_indices, page_size, raw_indices, True
)
# Candidate blocks one Triton program reduces; times block_size positions of logits.
_LEVEL_ONE_BLOCKS_PER_PROGRAM = 256
@triton.jit
def _candidate_block_scores_kernel(
logits_ptr,
seq_lens_ptr,
out_ptr,
logits_stride,
width,
num_blocks,
out_stride,
BLOCK_SIZE: tl.constexpr,
BLOCKS_PER_PROGRAM: tl.constexpr,
FILL_TAIL: tl.constexpr,
):
"""out[row, blk] = max(logits[row, blk * BLOCK_SIZE : (blk + 1) * BLOCK_SIZE])
over the positions < seq_lens[row]; +inf for the block holding the newest
position, -inf for blocks past the reach (written only with FILL_TAIL). A
program whose blocks all lie past the reach reads no logits."""
row = tl.program_id(0)
block0 = tl.program_id(1) * BLOCKS_PER_PROGRAM
length = tl.load(seq_lens_ptr + row)
blocks = block0 + tl.arange(0, BLOCKS_PER_PROGRAM)
in_table = blocks < num_blocks
if block0 * BLOCK_SIZE >= length:
if FILL_TAIL:
tl.store(out_ptr + row * out_stride + blocks, float("-inf"), mask=in_table)
return
cols = blocks[:, None] * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)[None, :]
vals = tl.load(
logits_ptr + row * logits_stride + cols,
mask=(cols < length) & (cols < width),
other=float("-inf"),
)
scores = tl.max(vals, axis=1)
last = (length - 1) // BLOCK_SIZE
scores = tl.where(blocks == last, float("inf"), scores)
tl.store(out_ptr + row * out_stride + blocks, scores, mask=in_table)
@triton.jit
def _gather_candidate_blocks_kernel(
logits_ptr,
seq_lens_ptr,
ids_ptr,
out_ptr,
logits_stride,
width,
ids_stride,
out_stride,
BLOCK_SIZE: tl.constexpr,
BLOCKS_PER_PROGRAM: tl.constexpr,
):
"""out[row, j * BLOCK_SIZE + t] = logits[row, ids[row, j] * BLOCK_SIZE + t] for
the reachable positions of the kept blocks, -inf elsewhere (an id of -1, or a
position past the reach in the newest block)."""
row = tl.program_id(0)
j0 = tl.program_id(1) * BLOCKS_PER_PROGRAM
length = tl.load(seq_lens_ptr + row)
j = j0 + tl.arange(0, BLOCKS_PER_PROGRAM)
ids = tl.load(ids_ptr + row * ids_stride + j)
t = tl.arange(0, BLOCK_SIZE)[None, :]
cols = ids[:, None] * BLOCK_SIZE + t
vals = tl.load(
logits_ptr + row * logits_stride + cols,
mask=(ids[:, None] >= 0) & (cols < length) & (cols < width),
other=float("-inf"),
)
tl.store(out_ptr + row * out_stride + j[:, None] * BLOCK_SIZE + t, vals)
def candidate_block_scores(
logits: torch.Tensor,
seq_lens: torch.Tensor,
*,
block_size: int,
fill_tail: bool,
) -> torch.Tensor:
"""[rows, num_blocks] fp32 block maxima of logits over the reachable positions
(see _candidate_block_scores_kernel). seq_lens int32 [rows], contiguous."""
assert logits.dim() == 2 and logits.dtype == torch.float32 and logits.stride(1) == 1
assert block_size & (block_size - 1) == 0, f"{block_size = } must be a power of 2"
rows, width = logits.shape
num_blocks = triton.cdiv(width, block_size)
scores = torch.empty((rows, num_blocks), dtype=torch.float32, device=logits.device)
grid = (rows, triton.cdiv(num_blocks, _LEVEL_ONE_BLOCKS_PER_PROGRAM))
_candidate_block_scores_kernel[grid](
logits,
seq_lens,
scores,
logits.stride(0),
width,
num_blocks,
scores.stride(0),
BLOCK_SIZE=block_size,
BLOCKS_PER_PROGRAM=_LEVEL_ONE_BLOCKS_PER_PROGRAM,
FILL_TAIL=fill_tail,
)
return scores
@triton.jit
def _candidate_lengths_kernel(
seq_lens_ptr,
block_lens_ptr,
compact_lens_ptr,
rows,
topk_blocks,
BLOCK_SIZE: tl.constexpr,
BLOCK: tl.constexpr,
):
"""block_lens = ceil(len / BLOCK_SIZE): the blocks with a reachable position;
compact_lens = min(block_lens, topk_blocks) * BLOCK_SIZE: the width of the
compact candidate row (every block is kept while there are at most
topk_blocks of them)."""
r = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)
mask = r < rows
length = tl.load(seq_lens_ptr + r, mask=mask, other=0)
block_lens = (length + (BLOCK_SIZE - 1)) // BLOCK_SIZE
tl.store(block_lens_ptr + r, block_lens, mask=mask)
tl.store(
compact_lens_ptr + r,
tl.minimum(block_lens, topk_blocks) * BLOCK_SIZE,
mask=mask,
)
@triton.jit
def _map_compact_selection_kernel(
compact_pos_ptr,
ids_ptr,
seq_lens_ptr,
page_table_ptr,
page_indices_ptr,
raw_indices_ptr,
ids_stride,
pt_stride,
out_stride,
n_pages,
BLOCK_SIZE: tl.constexpr,
PAGE_SIZE: tl.constexpr,
TOPK: tl.constexpr,
BLOCK: tl.constexpr,
WRITE_RAW: tl.constexpr,
SORT: tl.constexpr,
PAD_KEY: tl.constexpr,
):
"""Compact position c -> real position ids[c // BLOCK_SIZE] * BLOCK_SIZE + c % BLOCK_SIZE ->
slot via the row's page table; reachable selections packed first, -1 after. With SORT the valid
prefix is ascending by (position, slot)."""
row = tl.program_id(0)
length = tl.load(seq_lens_ptr + row)
j = tl.arange(0, BLOCK)
in_k = j < TOPK
c = tl.load(compact_pos_ptr + row * TOPK + j, mask=in_k, other=-1)
valid = c >= 0
cc = tl.where(valid, c, 0)
blk = tl.load(ids_ptr + row * ids_stride + cc // BLOCK_SIZE)
real = blk * BLOCK_SIZE + cc % BLOCK_SIZE
valid = valid & (blk >= 0) & (real < length)
page = tl.load(
page_table_ptr + row * pt_stride + tl.minimum(real // PAGE_SIZE, n_pages - 1),
mask=valid,
other=0,
)
slot = page * PAGE_SIZE + real % PAGE_SIZE
if SORT:
# BLOCK == TOPK (power of two): the sort key is the position with raw indices, else the slot
if WRITE_RAW:
hi = real
else:
hi = slot
key = tl.where(valid, hi, PAD_KEY).to(tl.int64) << 32
key = tl.sort(
key | (tl.where(valid, slot, -1).to(tl.int64) & 0xFFFFFFFF), dim=0
)
pad = (key >> 32) == PAD_KEY
tl.store(
page_indices_ptr + row * out_stride + j,
tl.where(pad, -1, (key & 0xFFFFFFFF).to(tl.int32)),
)
if WRITE_RAW:
tl.store(
raw_indices_ptr + row * out_stride + j,
tl.where(pad, -1, (key >> 32).to(tl.int32)),
)
else:
v = valid.to(tl.int32)
count = tl.sum(v, axis=0)
wpos = tl.cumsum(v, axis=0) - 1
# The packed entries and the padding never share an address.
tl.store(page_indices_ptr + row * out_stride + j, -1, mask=in_k & (j >= count))
tl.store(page_indices_ptr + row * out_stride + wpos, slot, mask=valid)
if WRITE_RAW:
tl.store(
raw_indices_ptr + row * out_stride + j, -1, mask=in_k & (j >= count)
)
tl.store(raw_indices_ptr + row * out_stride + wpos, real, mask=valid)
class CandidateBlocks(NamedTuple):
"""What the candidate-source layer publishes for the decode rows of a step."""
# int32 [rows, topk_blocks]: kept block ids in no particular order, -1 padded after the last
ids: torch.Tensor
# int32 [rows]: kept blocks * block_size, the width of the compact row.
compact_lens: torch.Tensor
# int32 [rows, 1] zeros; with page_size = compact_page_size a position maps to itself
compact_page_table: torch.Tensor
compact_page_size: int
block_size: int
def slice_candidate_blocks(candidates: CandidateBlocks, rows: slice) -> CandidateBlocks:
"""The rows rows of a per-request publication."""
return candidates._replace(
ids=candidates.ids[rows],
compact_lens=candidates.compact_lens[rows],
compact_page_table=candidates.compact_page_table[rows],
)
def cat_candidate_blocks(pieces: List[CandidateBlocks]) -> CandidateBlocks:
"""Row chunks of one request's publication, in row order."""
if len(pieces) == 1:
return pieces[0]
return pieces[0]._replace(
ids=torch.cat([p.ids for p in pieces]),
compact_lens=torch.cat([p.compact_lens for p in pieces]),
compact_page_table=torch.cat([p.compact_page_table for p in pieces]),
)
def select_candidate_blocks_hip(
logits: torch.Tensor,
seq_lens: torch.Tensor,
*,
topk_blocks: int,
block_size: int,
) -> CandidateBlocks:
"""Level one over a [rows, capacity] logits rectangle, bounded by seq_lens: the topk_blocks
best blocks of each row, the block with its newest position always among them; as a set per
row the ids equal the reference's select_candidate_blocks. Graph-safe: no host sync."""
seq_lens = seq_lens.to(torch.int32).contiguous()
rows, width = logits.shape
device = logits.device
num_blocks = triton.cdiv(width, block_size)
use_aot = topk_blocks == _AOT_FAST_TOPK_K
scores = candidate_block_scores(
logits, seq_lens, block_size=block_size, fill_tail=not use_aot
)
block_lens = torch.empty(rows, dtype=torch.int32, device=device)
compact_lens = torch.empty(rows, dtype=torch.int32, device=device)
_candidate_lengths_kernel[(triton.cdiv(rows, 1024),)](
seq_lens,
block_lens,
compact_lens,
rows,
topk_blocks,
BLOCK_SIZE=block_size,
BLOCK=1024,
)
if use_aot:
# exact top-k over the first ceil(len / block_size) block scores of each row, -1 padded
ids = torch.empty((rows, topk_blocks), dtype=torch.int32, device=device)
torch.ops.sgl_kernel.fast_topk(scores, ids, block_lens, None)
else:
picked = scores.topk(min(topk_blocks, num_blocks), dim=-1)
ids = picked.indices.to(torch.int32).masked_fill(
picked.values == -torch.inf, -1
)
# the gather takes the width from ids and tiles it in 256-block programs
ids = F.pad(ids, (0, topk_blocks - ids.shape[1]), value=-1)
compact_width = topk_blocks * block_size
return CandidateBlocks(
ids=ids,
compact_lens=compact_lens,
compact_page_table=torch.zeros((rows, 1), dtype=torch.int32, device=device),
compact_page_size=triton.next_power_of_2(compact_width),
block_size=block_size,
)
def gather_candidate_blocks(
logits: torch.Tensor,
seq_lens: torch.Tensor,
ids: torch.Tensor,
*,
block_size: int,
) -> torch.Tensor:
"""The compact [rows, topk_blocks * block_size] fp32 row of each request's
candidate positions in id order (see _gather_candidate_blocks_kernel)."""
rows, width = logits.shape
topk_blocks = ids.shape[1]
assert ids.shape[0] == rows and ids.dtype == torch.int32 and ids.stride(1) == 1
per_program = min(topk_blocks, _LEVEL_ONE_BLOCKS_PER_PROGRAM)
assert topk_blocks % per_program == 0, topk_blocks
compact = torch.empty(
(rows, topk_blocks * block_size), dtype=torch.float32, device=logits.device
)
grid = (rows, topk_blocks // per_program)
_gather_candidate_blocks_kernel[grid](
logits,
seq_lens.to(torch.int32).contiguous(),
ids,
compact,
logits.stride(0),
width,
ids.stride(0),
compact.stride(0),
BLOCK_SIZE=block_size,
BLOCKS_PER_PROGRAM=per_program,
)
return compact
def topk_within_candidate_blocks_hip(
logits: torch.Tensor,
seq_lens: torch.Tensor,
candidates: CandidateBlocks,
*,
page_table: torch.Tensor,
page_size: int,
page_indices: torch.Tensor,
raw_indices: Optional[torch.Tensor],
sort_output: bool = False,
) -> None:
"""Level two for a consumer layer: the top-k of logits inside the published candidate
blocks, written as the paged transform writes it (-1 padded, valid prefix first; ascending with
sort_output, k a power of two). Runs on the compact row, so the cost stops growing with context."""
rows, width = logits.shape
topk = page_indices.shape[1]
block_size = candidates.block_size
seq_lens = seq_lens.to(torch.int32).contiguous()
compact = gather_candidate_blocks(
logits, seq_lens, candidates.ids, block_size=block_size
)
assert compact.shape[1] <= candidates.compact_page_size
compact_pos = torch.empty((rows, topk), dtype=torch.int32, device=logits.device)
topk_transform_paged(
compact,
candidates.compact_lens,
candidates.compact_page_table,
compact_pos,
candidates.compact_page_size,
None,
)
assert page_indices.stride(1) == 1 and page_indices.shape == (rows, topk)
assert page_table.stride(1) == 1 and page_table.shape[0] == rows
write_raw = raw_indices is not None
if write_raw:
assert raw_indices.shape == (rows, topk) and raw_indices.stride(1) == 1
assert raw_indices.stride(0) == page_indices.stride(0)
assert not sort_output or topk & (topk - 1) == 0, topk
_map_compact_selection_kernel[(rows,)](
compact_pos,
candidates.ids,
seq_lens,
page_table,
page_indices,
raw_indices if write_raw else page_indices,
candidates.ids.stride(0),
page_table.stride(0),
page_indices.stride(0),
page_table.shape[1],
BLOCK_SIZE=block_size,
PAGE_SIZE=page_size,
TOPK=topk,
BLOCK=triton.next_power_of_2(topk),
WRITE_RAW=write_raw,
SORT=sort_output,
PAD_KEY=torch.iinfo(torch.int32).max,
)
@@ -0,0 +1,403 @@
"""Sparse attention reading V4.1 FP8/FP4 pages directly on HIP."""
from typing import Optional, Tuple
import torch
import triton
from aiter.ops.triton._gluon_kernels.gfx950.attention.pa_decode_sparse import _qkpv_fp8
from triton.experimental import gluon
from triton.experimental.gluon import language as gl
from sglang.kernels.ops.attention.dsv4.kv_layout import KVLayout
from sglang.kernels.ops.attention.sparse_decode_reduce_hip import (
gfx_sparse_split_reduce,
)
RCP_LN2 = gl.constexpr(1.4426950408889634) # exp(x) = exp2(x * RCP_LN2)
TILE_SLOTS = 64
MAX_SPLITS = 8
PACKED_TOKEN_BYTES = (KVLayout.V41.bytes_per_token, KVLayout.V41_FP4.bytes_per_token)
@gluon.jit
def _load_compact(Cache, offset, valid, USE_BUFFER: gl.constexpr):
if USE_BUFFER:
return gl.amd.cdna4.buffer_load(Cache, offset.to(gl.int32), valid, other=0)
return gl.load(Cache + offset, valid, 0)
@gluon.jit
def _read_compact_gluon(
Cache,
slots,
valid,
PAGE: gl.constexpr,
STRIDE: gl.constexpr,
FP4: gl.constexpr,
USE_BUFFER: gl.constexpr,
G: gl.constexpr,
):
d = gl.arange(0, 512, layout=gl.SliceLayout(0, G))
base = (slots // PAGE).to(gl.int64) * STRIDE
row = slots % PAGE
if FP4:
PG: gl.constexpr = gl.BlockedLayout([1, 8], [8, 8], [1, 4], [1, 0])
pd = gl.arange(0, 256, layout=gl.SliceLayout(0, PG))
packed_base = gl.convert_layout(base, gl.SliceLayout(1, PG))
packed_row = gl.convert_layout(row, gl.SliceLayout(1, PG))
packed_valid = gl.convert_layout(valid, gl.SliceLayout(1, PG))
packed = _load_compact(
Cache,
packed_base[:, None] + packed_row[:, None] * 256 + pd[None, :],
packed_valid[:, None],
USE_BUFFER,
)
value = gl.amd.cdna4.scaled_upcast(
packed,
gl.full((slots.shape[0], 512), 127, gl.uint8, G),
gl.bfloat16,
axis=1,
).to(gl.float32)
scale = (
_load_compact(
Cache,
base[:, None] + PAGE * 256 + row[:, None] * 32 + d[None, :] // 16,
valid[:, None],
USE_BUFFER,
)
.to(gl.float8e4nv, bitcast=True)
.to(gl.float32)
)
else:
value = (
_load_compact(
Cache,
base[:, None] + row[:, None] * 512 + d[None, :],
valid[:, None],
USE_BUFFER,
)
.to(gl.float8e4nv, bitcast=True)
.to(gl.float32)
)
exp = _load_compact(
Cache,
base[:, None] + PAGE * 512 + row[:, None] * 16 + d[None, :] // 32,
valid[:, None],
USE_BUFFER,
).to(gl.int32)
scale = gl.where(exp == 0, 0x00400000, exp << 23).to(gl.float32, bitcast=True)
scale = gl.where(exp == 255, float("nan"), scale)
return (value * scale).to(gl.bfloat16)
@gluon.jit
def _compact_qkpv(
kv,
valid,
qdot,
m,
den,
acc,
head_mask,
smem,
MF: gl.constexpr,
H: gl.constexpr,
BLOCK: gl.constexpr,
SCALE: gl.constexpr,
):
# aiter's private helper, by keyword: a renamed or reordered parameter fails to compile
return _qkpv_fp8(
k_nope=kv,
k_rope=kv,
valid=valid,
q_dot=qdot,
m_i=m,
l_i=den,
acc=acc,
head_mask=head_mask,
qk_scale=SCALE * RCP_LN2,
kv_smem=smem,
qk_layout=MF,
pv_layout=MF,
k_layout=gl.DotOperandLayout(1, MF, 8),
v_layout=gl.DotOperandLayout(1, MF, 8),
p_layout=gl.DotOperandLayout(0, MF, 8),
NOPE_DIM=512,
ROPE_DIM=0,
HEAD_SIZE=512,
BLOCK_M=16,
BLOCK_K=BLOCK,
HEAD_ALIGNED=H % 16 == 0,
UNIFORM=True,
HAS_INVALID=True,
)
@gluon.jit
def _compact_tile(
Cache,
Indices,
t,
start,
length,
STRIDE: gl.constexpr,
CAPACITY: gl.constexpr,
PAGE: gl.constexpr,
CACHE_STRIDE: gl.constexpr,
FP4: gl.constexpr,
BUFFER: gl.constexpr,
G: gl.constexpr,
BLOCK: gl.constexpr,
):
col = start + gl.arange(0, BLOCK, layout=gl.SliceLayout(1, G))
slot = gl.load(Indices + t * STRIDE + col, col < length, -1)
valid = (col < length) & (slot >= 0) & (slot < CAPACITY)
kv = _read_compact_gluon(
Cache, gl.where(valid, slot, 0), valid, PAGE, CACHE_STRIDE, FP4, BUFFER, G
)
return kv, valid
@gluon.jit
def _compact_attention_kernel(
Q,
K,
E,
I,
EI,
L,
EL,
Acc,
Max,
Sum,
Out,
Sink,
Freqs,
Positions,
HAS_SINK: gl.constexpr,
ROPE_DIM: gl.constexpr,
FREQ_STRIDE: gl.constexpr,
QS: gl.constexpr,
QH: gl.constexpr,
H: gl.constexpr,
KS: gl.constexpr,
ES: gl.constexpr,
KP: gl.constexpr,
EP: gl.constexpr,
KN: gl.constexpr,
EN: gl.constexpr,
IS: gl.constexpr,
EIS: gl.constexpr,
NK: gl.constexpr,
NE: gl.constexpr,
KFP4: gl.constexpr,
EFP4: gl.constexpr,
KBUFFER: gl.constexpr,
EBUFFER: gl.constexpr,
SPLITS: gl.constexpr,
SCALE: gl.constexpr,
BLOCK: gl.constexpr,
):
t, split, hg = gl.program_id(0).to(gl.int64), gl.program_id(1), gl.program_id(2)
MF: gl.constexpr = gl.amd.AMDMFMALayout(
version=4, instr_shape=[16, 16, 16], transposed=True, warps_per_cta=[1, 4]
)
G: gl.constexpr = gl.BlockedLayout([1, 16], [8, 8], [1, 4], [1, 0])
QG: gl.constexpr = gl.BlockedLayout([1, 8], [8, 8], [1, 4], [1, 0])
hq = hg * 16 + gl.arange(0, 16, layout=gl.SliceLayout(1, QG))
dq = gl.arange(0, 512, layout=gl.SliceLayout(0, QG))
q = gl.load(Q + t * QS + hq[:, None] * QH + dq[None, :], hq[:, None] < H, 0)
qdot = gl.convert_layout(q, gl.DotOperandLayout(0, MF, 8))
h = hg * 16 + gl.arange(0, 16, layout=gl.SliceLayout(1, MF))
d = gl.arange(0, 512, layout=gl.SliceLayout(0, MF))
m = gl.full((16,), -float("inf"), gl.float32, layout=gl.SliceLayout(1, MF))
den = gl.zeros((16,), gl.float32, layout=gl.SliceLayout(1, MF))
acc = gl.zeros((16, 512), gl.float32, layout=MF)
smem = gl.allocate_shared_memory(
gl.bfloat16,
[BLOCK, 512],
gl.PaddedSharedLayout.with_identity_for([[512, 8]], [BLOCK, 512], [1, 0]),
)
for segment in gl.static_range(2):
if segment == 0 or NE > 0:
if segment == 0:
indices, lengths, width, stride, capacity = I, L, NK, IS, KN
cache, page, cache_stride, fp4, buffer = K, KP, KS, KFP4, KBUFFER
else:
indices, lengths, width, stride, capacity = EI, EL, NE, EIS, EN
cache, page, cache_stride, fp4, buffer = E, EP, ES, EFP4, EBUFFER
length = gl.minimum(gl.load(lengths + t), width)
if split * BLOCK < length:
kv, valid = _compact_tile(
cache,
indices,
t,
split * BLOCK,
length,
stride,
capacity,
page,
cache_stride,
fp4,
buffer,
G,
BLOCK,
)
for start in range((split + SPLITS) * BLOCK, length, SPLITS * BLOCK):
next_kv, next_valid = _compact_tile(
cache,
indices,
t,
start,
length,
stride,
capacity,
page,
cache_stride,
fp4,
buffer,
G,
BLOCK,
)
m, den, acc = _compact_qkpv(
kv, valid, qdot, m, den, acc, h < H, smem, MF, H, BLOCK, SCALE
)
kv, valid = next_kv, next_valid
m, den, acc = _compact_qkpv(
kv, valid, qdot, m, den, acc, h < H, smem, MF, H, BLOCK, SCALE
)
if SPLITS == 1:
if HAS_SINK:
sink = gl.load(Sink + h, h < H, -float("inf"))
scaled_max = m * (SCALE * RCP_LN2)
final_max = gl.maximum(scaled_max, sink * RCP_LN2)
weight = gl.exp2(scaled_max - final_max)
den = den * weight + gl.exp2(sink * RCP_LN2 - final_max)
acc = acc * weight[:, None]
out = (acc / den[:, None]).to(gl.bfloat16)
out = gl.convert_layout(out, QG)
if ROPE_DIM > 0:
pos = gl.load(Positions + t)
is_rope = dq >= 512 - ROPE_DIM
freq_index = ((dq - (512 - ROPE_DIM)) // 2) * 2
cos = gl.load(Freqs + pos * FREQ_STRIDE + freq_index, is_rope, 0)
sin = gl.load(Freqs + pos * FREQ_STRIDE + freq_index + 1, is_rope, 0)
x = out.to(gl.float32)
even, odd = gl.split(gl.reshape(x, (16, 256, 2)))
swapped = gl.reshape(gl.join(odd, -even), (16, 512))
rotated = gl.fma(x, cos[None, :], swapped * sin[None, :])
out = gl.where(is_rope[None, :], rotated.to(gl.bfloat16), out)
gl.store(Out + (t * H + hq[:, None]) * 512 + dq[None, :], out, hq[:, None] < H)
else:
off = (t * SPLITS + split) * H + h
gl.store(Max + off, m * (SCALE * RCP_LN2), h < H)
gl.store(Sum + off, den, h < H)
gl.store(Acc + off[:, None] * 512 + d[None, :], acc, h[:, None] < H)
def compact_attention_hip(
q: torch.Tensor,
cache: torch.Tensor,
indices: torch.Tensor,
lengths: torch.Tensor,
sink: Optional[torch.Tensor],
*,
extra_cache: Optional[torch.Tensor] = None,
extra_indices: Optional[torch.Tensor] = None,
extra_lengths: Optional[torch.Tensor] = None,
softmax_scale: float = 512**-0.5,
splits: Optional[int] = None,
inv_rope: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
) -> torch.Tensor:
"""q [n, h, 512] bf16 over the first lengths[t] slots of indices[t] (-1 skipped) in the
packed V41 / V41_FP4 cache [pages, page_size, 1, bytes_per_token], then those of
extra_indices[t] in extra_cache, the sink folded in once. inv_rope is
(freqs_real, positions): the tail of every head is inverse-rotated."""
assert q.ndim == 3 and q.shape[-1] == 512 and q.dtype == torch.bfloat16
assert q.stride(-1) == 1
assert cache.ndim == 4 and cache.shape[2] == 1
assert cache.shape[-1] in PACKED_TOKEN_BYTES
n, h, _ = q.shape
if n == 0:
return torch.empty_like(q)
indices = indices.reshape(n, -1)
assert indices.stride(-1) == 1 and lengths.is_contiguous()
if extra_cache is None:
assert extra_indices is None and extra_lengths is None
extra_cache, extra_indices, extra_lengths = cache, indices, lengths
ne = 0
else:
assert extra_cache.shape[-1] in PACKED_TOKEN_BYTES
extra_indices = extra_indices.reshape(n, -1)
assert extra_indices.stride(-1) == 1 and extra_lengths.is_contiguous()
ne = extra_indices.shape[1]
if splits is None:
# enough splits to fill the CUs, none past the slots a row has
num_cus = torch.cuda.get_device_properties(q.device).multi_processor_count
splits = min(
MAX_SPLITS,
triton.cdiv(indices.shape[1] + ne, TILE_SLOTS),
triton.cdiv(num_cus, n * triton.cdiv(h, 16)),
)
assert splits > 0
out = torch.empty((n, h, 512), dtype=q.dtype, device=q.device) if splits == 1 else q
if inv_rope is not None:
freqs, positions = inv_rope
assert freqs.dtype == torch.float32 and freqs.stride(1) == 1
assert positions.shape == (n,)
rope_dim = freqs.shape[1]
assert 0 < rope_dim <= 512 and rope_dim % 2 == 0
else:
freqs = positions = out
rope_dim = 0
if splits == 1:
acc = maximum = denominator = out
else:
acc = torch.empty((n, splits, h, 512), dtype=torch.float32, device=q.device)
maximum = torch.empty((n, splits, h), dtype=torch.float32, device=q.device)
denominator = torch.empty_like(maximum)
_compact_attention_kernel[(n, splits, triton.cdiv(h, 16))](
q,
cache.view(torch.uint8),
extra_cache.view(torch.uint8),
indices,
extra_indices,
lengths,
extra_lengths,
acc,
maximum,
denominator,
out,
sink if sink is not None else out,
freqs,
positions,
HAS_SINK=sink is not None,
ROPE_DIM=rope_dim,
FREQ_STRIDE=freqs.stride(0),
QS=q.stride(0),
QH=q.stride(1),
H=h,
KS=cache.stride(0),
ES=extra_cache.stride(0),
KP=cache.shape[1],
EP=extra_cache.shape[1],
KN=cache.shape[0] * cache.shape[1],
EN=extra_cache.shape[0] * extra_cache.shape[1],
IS=indices.stride(0),
EIS=extra_indices.stride(0),
NK=indices.shape[1],
NE=ne,
KFP4=cache.shape[-1] == KVLayout.V41_FP4.bytes_per_token,
EFP4=extra_cache.shape[-1] == KVLayout.V41_FP4.bytes_per_token,
KBUFFER=cache.shape[0] * cache.stride(0) < 2**31,
EBUFFER=extra_cache.shape[0] * extra_cache.stride(0) < 2**31,
SPLITS=splits,
SCALE=softmax_scale,
BLOCK=TILE_SLOTS,
num_warps=4,
)
if splits == 1:
return out
return gfx_sparse_split_reduce(
acc, maximum, denominator, sink, q.dtype, inv_rope=inv_rope
)
@@ -1,5 +1,5 @@
"""SM100 small-batch paged attention with the heads on the MMA N dimension; the
caller applies the inverse RoPE to the result."""
inverse RoPE is applied by the caller, or by the combine when inv_rope is given."""
from typing import Optional
@@ -37,6 +37,23 @@ def can_use_swapab_attention(
)
@triton.jit
def inverse_rope_tail(
out, d, FREQS, pos, freq_stride, HEAD_DIM: tl.constexpr, ROPE_DIM: tl.constexpr
):
"""out (head dims d) with its last ROPE_DIM dims inverse-rotated at position pos. It
rotates the rounded values, as the model's standalone RoPE kernel reads them."""
x = out.to(tl.float32)
is_rope = d >= HEAD_DIM - ROPE_DIM
freq_index = ((d - (HEAD_DIM - ROPE_DIM)) // 2) * 2
cos = tl.load(FREQS + pos * freq_stride + freq_index, is_rope, 0)
sin = tl.load(FREQS + pos * freq_stride + freq_index + 1, is_rope, 0)
signed = tl.where(d % 2 == 0, -x * sin, x * sin)
swapped = tl.reshape(tl.flip(tl.reshape(signed, (d.shape[0] // 2, 2)), 1), d.shape)
rotated = tl.fma(x, cos, swapped)
return tl.where(is_rope, rotated.to(out.dtype), out)
@triton.jit
def _combine(
PART,
@@ -44,6 +61,10 @@ def _combine(
SUM,
SINK,
OUT,
FREQS,
POSITIONS,
FREQ_STRIDE: tl.constexpr,
ROPE_DIM: tl.constexpr,
NT: tl.constexpr,
ST: tl.constexpr,
H: tl.constexpr,
@@ -66,6 +87,10 @@ def _combine(
)
out = tl.sum(vals * factor[:, None], 0) / denominator
out = tl.where((denominator > 0) & (sink != float("inf")), out, 0.0)
out = out.to(OUT.dtype.element_ty)
if ROPE_DIM > 0:
pos = tl.load(POSITIONS + b)
out = inverse_rope_tail(out, d, FREQS, pos, FREQ_STRIDE, 512, ROPE_DIM)
tl.store(OUT + (b * H + h) * 512 + d, out)
@@ -78,10 +103,14 @@ def swapab_attention(
extra_kv=None,
extra_indices=None,
extra_lengths=None,
inv_rope=None,
partial_gluon=None,
):
"""V4-layout attention on 16 heads; `extra_*` is a second slot range appended
to each request's keys, and the attention sink is folded in exactly once."""
from .decode_attention_sm100_gluon import partial_gluon
to each request's keys, and the attention sink is folded in exactly once.
`partial_gluon` replaces the SM100 split-KV kernel (swapab_gluon_hip passes gfx950's)."""
if partial_gluon is None:
from .decode_attention_sm100_gluon import partial_gluon
block = 64
b, h, d = q.shape[0], q.shape[-2], q.shape[-1]
@@ -144,6 +173,15 @@ def swapab_attention(
TILE=LAYOUT.tile_size,
num_warps=4,
)
if inv_rope is not None:
freqs, positions = inv_rope
assert freqs.dtype == torch.float32 and freqs.stride(1) == 1
assert positions.shape == (b,)
rope_dim = freqs.shape[1]
assert 0 < rope_dim <= 512 and rope_dim % 2 == 0
else:
freqs = positions = out
rope_dim = 0
bd = 64 if ne else 512
_combine[(b, h, triton.cdiv(512, bd))](
partial,
@@ -151,6 +189,10 @@ def swapab_attention(
sums,
sink,
out,
freqs,
positions,
FREQ_STRIDE=freqs.stride(0),
ROPE_DIM=rope_dim,
NT=nt,
ST=triton.next_power_of_2(nt),
H=h,
@@ -5,11 +5,19 @@ import triton
import triton.language as tl
from sglang.kernels.jit.utils import is_arch_support_pdl
from sglang.srt.utils import is_hip
@triton.jit
def _q_rope_store(
X, Y, F, POS, SX: tl.constexpr, SY: tl.constexpr, USE_GDC: tl.constexpr = False
X,
Y,
F,
POS,
SX: tl.constexpr,
SY: tl.constexpr,
IS_HIP: tl.constexpr,
USE_GDC: tl.constexpr = False,
):
row, head = tl.program_id(0), tl.program_id(1)
r = tl.arange(0, 512)
@@ -22,7 +30,11 @@ def _q_rope_store(
sin = tl.load(F + position * 64 + (r - 448) // 2 * 2 + 1, r >= 448, 0)
# Match the operation order of deepseek_rope_kernel before BF16 rounding.
even = tl.fma(value, cos, -partner * sin)
odd = tl.fma(partner, sin, value * cos)
if IS_HIP:
# Match AMD flat RoPE: round the sine product before the cosine FMA.
odd = tl.fma(value, cos, partner * sin)
else:
odd = tl.fma(partner, sin, value * cos)
rotated = tl.where((r & 1) == 0, even, odd)
tl.store(Y + row * SY + head * 512 + r, tl.where(r >= 448, rotated, value))
if USE_GDC:
@@ -40,6 +52,7 @@ def _q_rope_store_prefill(
SX: tl.constexpr,
SY: tl.constexpr,
BLOCK_HEADS: tl.constexpr,
IS_HIP: tl.constexpr,
):
# Keep token count dynamic to avoid compiling every prefill batch length.
heads = tl.program_id(0) * BLOCK_HEADS + tl.arange(0, BLOCK_HEADS)
@@ -56,7 +69,11 @@ def _q_rope_store_prefill(
sin = tl.load(F + freq_offset + 1, rope_mask, 0)
# Keep the same arithmetic and BF16 rounding as the decode kernel.
even = tl.fma(value, cos, -partner * sin)
odd = tl.fma(partner, sin, value * cos)
if IS_HIP:
# Match AMD flat RoPE: round the sine product before the cosine FMA.
odd = tl.fma(value, cos, partner * sin)
else:
odd = tl.fma(partner, sin, value * cos)
rotated = tl.where((r[None, :] & 1) == 0, even, odd)
y_offset = row[:, None].to(tl.int64) * SY + head[:, None] * 512 + r[None, :]
tl.store(
@@ -91,6 +108,7 @@ def q_rope_store(
q.stride(0),
output.stride(0),
BLOCK_HEADS=4,
IS_HIP=is_hip(),
num_warps=4,
)
return
@@ -102,6 +120,7 @@ def q_rope_store(
positions,
q.stride(0),
output.stride(0),
IS_HIP=is_hip(),
num_warps=4,
**pdl_kwargs,
)
@@ -0,0 +1,130 @@
"""Native 16-head MFMA layouts for gfx950 paged attention."""
import functools
from triton.experimental import gluon
from triton.experimental.gluon import language as gl
from .decode_attention_sm100 import swapab_attention as _swapab_attention
# the V4 page load is plain gluon (no Blackwell ops), so the SM100 kernel's serves gfx950 too
from .decode_attention_sm100_gluon import _load_v4
@gluon.jit
def partial_gluon(
Q,
K,
E,
IDX,
EI,
L,
EL,
PART,
MAX,
SUM,
QS: gl.constexpr,
QH: gl.constexpr,
IS: gl.constexpr,
EIS: gl.constexpr,
KP: gl.constexpr,
KS: gl.constexpr,
EP: gl.constexpr,
ES: gl.constexpr,
NK: gl.constexpr,
NE: gl.constexpr,
NT: gl.constexpr,
KT: gl.constexpr,
BT: gl.constexpr,
H: gl.constexpr,
SCALE: gl.constexpr,
KTOKENS: gl.constexpr,
ETOKENS: gl.constexpr,
COMPENSATE: gl.constexpr,
SWAP_AB: gl.constexpr,
DATA_BYTES: gl.constexpr,
SCALE_BYTES: gl.constexpr,
TILE: gl.constexpr,
):
gl.static_assert(SWAP_AB and H == 16 and (BT == 64 or BT == 128))
b, t = gl.program_id(0), gl.program_id(1)
kv_layout: gl.constexpr = gl.BlockedLayout([1, 8], [8, 8], [1, 4], [1, 0])
n = gl.arange(0, BT, gl.SliceLayout(1, kv_layout))
if t < KT:
at = t * BT + n
length = gl.load(L + b)
ids = gl.load(IDX + b * IS + at, at < NK, -1)
valid = (at < NK) & (at < length) & (ids >= 0) & (ids < KTOKENS)
kv = _load_v4(
K,
gl.maximum(ids, 0),
valid,
KP,
KS,
kv_layout,
DATA_BYTES,
SCALE_BYTES,
TILE,
)
else:
at = (t - KT) * BT + n
length = gl.load(EL + b)
ids = gl.load(EI + b * EIS + at, at < NE, -1)
valid = (at < NE) & (at < length) & (ids >= 0) & (ids < ETOKENS)
kv = _load_v4(
E,
gl.maximum(ids, 0),
valid,
EP,
ES,
kv_layout,
DATA_BYTES,
SCALE_BYTES,
TILE,
)
qh = gl.arange(0, H, gl.SliceLayout(1, kv_layout))
qd = gl.arange(0, 512, gl.SliceLayout(0, kv_layout))
q = gl.load(Q + b * QS + qh[:, None] * QH + qd[None, :])
mf: gl.constexpr = gl.amd.AMDMFMALayout(
version=4, instr_shape=[16, 16, 16], transposed=True, warps_per_cta=[4, 1]
)
scores = (
gl.amd.cdna4.mfma(
gl.convert_layout(kv, gl.DotOperandLayout(0, mf, 8)),
gl.convert_layout(q.permute((1, 0)), gl.DotOperandLayout(1, mf, 8)),
gl.zeros((BT, H), gl.float32, mf),
)
* SCALE
)
valid = gl.convert_layout(valid, gl.SliceLayout(1, mf))
scores = gl.where(valid[:, None], scores, -float("inf"))
mx = gl.max(scores, 0)
mx = gl.where(mx == -float("inf"), 0.0, mx)
prob = gl.exp(scores - mx[None, :])
denom = gl.sum(prob, 0)
hi = prob.to(gl.bfloat16)
value = gl.amd.cdna4.mfma(
gl.convert_layout(kv.permute((1, 0)), gl.DotOperandLayout(0, mf, 8)),
gl.convert_layout(hi, gl.DotOperandLayout(1, mf, 8)),
gl.zeros((512, H), gl.float32, mf),
)
if COMPENSATE:
lo = (prob - hi.to(gl.float32)).to(gl.bfloat16)
value = gl.amd.cdna4.mfma(
gl.convert_layout(kv.permute((1, 0)), gl.DotOperandLayout(0, mf, 8)),
gl.convert_layout(lo, gl.DotOperandLayout(1, mf, 8)),
value,
)
hd: gl.constexpr = gl.BlockedLayout([1, 8], [8, 8], [1, 4], [1, 0])
value = gl.convert_layout(value.permute((1, 0)), hd)
h = gl.arange(0, H, gl.SliceLayout(1, hd))
d = gl.arange(0, 512, gl.SliceLayout(0, hd))
gl.store(PART + ((b * NT + t) * H + h[:, None]) * 512 + d[None, :], value)
stat: gl.constexpr = gl.BlockedLayout([1], [64], [4], [0])
hs = gl.arange(0, H, stat)
gl.store(MAX + (b * NT + t) * H + hs, gl.convert_layout(mx, stat))
gl.store(SUM + (b * NT + t) * H + hs, gl.convert_layout(denom, stat))
# the SM100 host path (sink, combine, inverse RoPE) on the gfx950 split-KV kernel above
swapab_attention = functools.partial(_swapab_attention, partial_gluon=partial_gluon)
@@ -0,0 +1,133 @@
# SPDX-License-Identifier: Apache-2.0
"""Split-KV combine for aiter's gfx950 sparse decode kernel, bitwise _pa_decode_sparse_reduce,
with the DeepSeek-V4 inverse RoPE folded in."""
from __future__ import annotations
from typing import Optional, Tuple
import torch
import triton
import triton.language as tl
from sglang.kernels.ops.attention.dsv4.decode_attention_sm100 import inverse_rope_tail
@triton.jit
def _gfx_sparse_decode_reduce_kernel(
part_m_ptr,
part_l_ptr,
part_acc_ptr,
sink_ptr,
out_ptr,
fr_ptr,
pos_ptr,
pm_stride_t,
pm_stride_s,
pa_stride_t,
pa_stride_s,
pa_stride_h,
out_stride_t,
out_stride_h,
fr_stride_pos,
NUM_SPLITS: tl.constexpr,
D: tl.constexpr,
RD: tl.constexpr,
HAS_SINK: tl.constexpr,
INV_ROPE: tl.constexpr,
):
"""Grid (num_queries, num_heads). Partials: part_m / part_l
[T, S, H] fp32 (m in the base-2 exponent domain), part_acc
[T, S, H, D] fp32 -- the layout aiter's gluon main kernel stores."""
RCP_LN2: tl.constexpr = 1.4426950408889634
t = tl.program_id(0)
h = tl.program_id(1)
d = tl.arange(0, D)
pm_base = t * pm_stride_t + h
pa_base = t * pa_stride_t + h * pa_stride_h
neg_inf = float("-inf")
m_final = tl.full((), neg_inf, tl.float32)
for s in tl.static_range(NUM_SPLITS):
m_s = tl.load(part_m_ptr + pm_base + s * pm_stride_s)
m_final = tl.maximum(m_final, m_s)
if HAS_SINK:
sink = tl.load(sink_ptr + h).to(tl.float32)
m_final = tl.maximum(m_final, sink * RCP_LN2)
l_final = tl.full((), 0.0, tl.float32)
acc = tl.zeros([D], dtype=tl.float32)
for s in tl.static_range(NUM_SPLITS):
m_s = tl.load(part_m_ptr + pm_base + s * pm_stride_s)
l_s = tl.load(part_l_ptr + pm_base + s * pm_stride_s)
w = tl.exp2(m_s - m_final)
# gluon contracts l + w * l_s into one fma; Triton leaves the scalar chain as mul + add
l_final = tl.fma(w, l_s, l_final)
acc_s = tl.load(part_acc_ptr + pa_base + s * pa_stride_s + d)
acc = tl.fma(w, acc_s, acc)
if HAS_SINK:
l_final = l_final + tl.exp2(sink * RCP_LN2 - m_final)
out = (acc / l_final).to(out_ptr.dtype.element_ty)
if INV_ROPE:
pos = tl.load(pos_ptr + t)
out = inverse_rope_tail(out, d, fr_ptr, pos, fr_stride_pos, D, RD)
tl.store(out_ptr + t * out_stride_t + h * out_stride_h + d, out)
def gfx_sparse_split_reduce(
part_acc: torch.Tensor,
part_m: torch.Tensor,
part_l: torch.Tensor,
attn_sink: Optional[torch.Tensor],
out_dtype: torch.dtype = torch.bfloat16,
inv_rope: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
) -> torch.Tensor:
"""Combine pa_decode_sparse(..., skip_reduce=True)'s partials into the
[T, H, D] output. inv_rope is (freqs_real [max_pos, rope_dim] fp32,
positions [T]): the last rope_dim of every head are inverse-rotated."""
T, S, H, D = part_acc.shape
assert part_m.shape == (T, S, H) and part_l.shape == (T, S, H), (
part_m.shape,
part_l.shape,
)
assert part_acc.stride(3) == 1 and part_m.stride(2) == 1 and part_l.stride(2) == 1
assert part_m.stride() == part_l.stride()
assert triton.next_power_of_2(D) == D and S >= 1
out = torch.empty((T, H, D), dtype=out_dtype, device=part_acc.device)
if T == 0:
return out
has_sink = attn_sink is not None
if inv_rope is not None:
freqs_real, positions = inv_rope
rope_dim = freqs_real.shape[1]
assert freqs_real.dtype == torch.float32 and freqs_real.stride(1) == 1
assert positions.shape == (T,), positions.shape
assert rope_dim % 2 == 0 and rope_dim <= D
else:
freqs_real = positions = out # unread placeholders
rope_dim = 0
_gfx_sparse_decode_reduce_kernel[(T, H)](
part_m,
part_l,
part_acc,
attn_sink if has_sink else out,
out,
freqs_real,
positions,
part_m.stride(0),
part_m.stride(1),
part_acc.stride(0),
part_acc.stride(1),
part_acc.stride(2),
out.stride(0),
out.stride(1),
freqs_real.stride(0),
NUM_SPLITS=S,
D=D,
RD=rope_dim,
HAS_SINK=has_sink,
INV_ROPE=inv_rope is not None,
num_warps=1,
)
return out