From 096b066fb4f6dc3b058ff294cb9e8f7f5a7150cf Mon Sep 17 00:00:00 2001 From: Kevin Mi <45493463+kevin-mii@users.noreply.github.com> Date: Mon, 28 Sep 2026 16:14:28 -0700 Subject: [PATCH] dsv4.1-amd: gfx950 sparse decode attention and sorted top-k (#41020) Co-authored-by: Claude Opus 5.5 --- .../benchmark/bench_dsv4_topk_transform.py | 76 ++++ .../kernels/aot/csrc/common_extension_rocm.cc | 2 +- .../aot/csrc/elementwise/deepseek_v4_topk.cu | 178 +++++++- .../kernels/aot/include/sgl_kernel_ops.h | 3 +- .../kernels/aot/python/sgl_kernel/top_k.py | 9 +- python/sglang/kernels/aot/tests/test_topk.py | 45 ++ .../ops/attention/dsv4/attn_glue_hip.py | 426 ++++++++++++++++++ .../attention/dsv4/candidate_blocks_hip.py | 408 +++++++++++++++++ .../attention/dsv4/compact_attention_hip.py | 403 +++++++++++++++++ .../attention/dsv4/decode_attention_sm100.py | 48 +- .../ops/attention/dsv4/q_rope_store.py | 25 +- .../ops/attention/dsv4/swapab_gluon_hip.py | 130 ++++++ .../ops/attention/sparse_decode_reduce_hip.py | 133 ++++++ .../dsv4/test_compact_attention_hip.py | 103 +++++ .../dsv4/test_dsv41_decode_glue_hip.py | 358 +++++++++++++++ .../attention/dsv4/test_swapab_attention.py | 227 ++++++++++ .../ops/attention/test_dsv4_q_rope_store.py | 12 +- 17 files changed, 2569 insertions(+), 17 deletions(-) create mode 100644 python/sglang/kernels/aot/benchmark/bench_dsv4_topk_transform.py create mode 100644 python/sglang/kernels/ops/attention/dsv4/attn_glue_hip.py create mode 100644 python/sglang/kernels/ops/attention/dsv4/candidate_blocks_hip.py create mode 100644 python/sglang/kernels/ops/attention/dsv4/compact_attention_hip.py create mode 100644 python/sglang/kernels/ops/attention/dsv4/swapab_gluon_hip.py create mode 100644 python/sglang/kernels/ops/attention/sparse_decode_reduce_hip.py create mode 100644 test/registered/kernels/ops/attention/dsv4/test_compact_attention_hip.py create mode 100644 test/registered/kernels/ops/attention/dsv4/test_dsv41_decode_glue_hip.py create mode 100644 test/registered/kernels/ops/attention/dsv4/test_swapab_attention.py diff --git a/python/sglang/kernels/aot/benchmark/bench_dsv4_topk_transform.py b/python/sglang/kernels/aot/benchmark/bench_dsv4_topk_transform.py new file mode 100644 index 0000000000..8db36471b5 --- /dev/null +++ b/python/sglang/kernels/aot/benchmark/bench_dsv4_topk_transform.py @@ -0,0 +1,76 @@ +"""Benchmark for the DeepSeek-V4 indexer top-k transform, with and without sorted output.""" + +import itertools + +import torch +import triton +import triton.testing +from sgl_kernel import deepseek_v4_topk_transform_512 + +try: + from sglang.utils import is_in_ci + + IS_CI = is_in_ci() +except ImportError: + IS_CI = False + +TOPK = 512 +PAGE_SIZE = 64 + +batch_sizes = [1] if IS_CI else [1, 8, 32, 128] +seq_lens = [4096] if IS_CI else [1024, 4096, 32768] + +configs = list(itertools.product(batch_sizes, seq_lens)) + + +@triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["batch_size", "seq_len"], + x_vals=configs, + line_arg="provider", + line_vals=["unsorted", "sorted", "torch_sort"], + line_names=["SGL Kernel", "SGL Kernel sort_output", "SGL Kernel + torch.sort"], + styles=[("blue", "-"), ("green", "-"), ("red", "--")], + ylabel="µs (median)", + plot_name="dsv4-topk-transform-performance", + args={}, + ) +) +def benchmark_topk_transform(batch_size, seq_len, provider): + torch.manual_seed(42) + scores = torch.randn(batch_size, seq_len, dtype=torch.float32, device="cuda") + lengths = torch.full((batch_size,), seq_len, dtype=torch.int32, device="cuda") + num_pages = triton.cdiv(seq_len, PAGE_SIZE) + page_table = torch.stack( + [ + torch.randperm(4 * num_pages, device="cuda")[:num_pages] + for _ in range(batch_size) + ] + ).int() + page_indices = torch.empty(batch_size, TOPK, dtype=torch.int32, device="cuda") + + def topk(sort_output): + deepseek_v4_topk_transform_512( + scores, + lengths, + page_table, + page_indices, + PAGE_SIZE, + sort_output=sort_output, + ) + + if provider == "unsorted": + fn = lambda: topk(False) + elif provider == "sorted": + fn = lambda: topk(True) + else: + fn = lambda: (topk(False), torch.sort(page_indices, dim=-1)) + + ms, min_ms, max_ms = triton.testing.do_bench_cudagraph( + fn, quantiles=[0.5, 0.2, 0.8] + ) + return 1000 * ms, 1000 * max_ms, 1000 * min_ms + + +if __name__ == "__main__": + benchmark_topk_transform.run(print_data=True) diff --git a/python/sglang/kernels/aot/csrc/common_extension_rocm.cc b/python/sglang/kernels/aot/csrc/common_extension_rocm.cc index cce64fd578..fa2d5a7cfb 100644 --- a/python/sglang/kernels/aot/csrc/common_extension_rocm.cc +++ b/python/sglang/kernels/aot/csrc/common_extension_rocm.cc @@ -49,7 +49,7 @@ TORCH_LIBRARY_EXPAND(sgl_kernel, m) { m.def( "deepseek_v4_topk_transform_512(Tensor scores, Tensor seq_lens, Tensor page_table, Tensor! " - "page_indices, int page_size, Tensor!? raw_indices) -> ()"); + "page_indices, int page_size, Tensor!? raw_indices, bool sort_output=False) -> ()"); m.impl("deepseek_v4_topk_transform_512", torch::kCUDA, &deepseek_v4_topk_transform_512); m.def( diff --git a/python/sglang/kernels/aot/csrc/elementwise/deepseek_v4_topk.cu b/python/sglang/kernels/aot/csrc/elementwise/deepseek_v4_topk.cu index 0a1e982996..94048a6055 100644 --- a/python/sglang/kernels/aot/csrc/elementwise/deepseek_v4_topk.cu +++ b/python/sglang/kernels/aot/csrc/elementwise/deepseek_v4_topk.cu @@ -46,6 +46,7 @@ constexpr size_t kSMEM = 48 * 1024; // bytes #endif static_assert(kSMEM % (2 * sizeof(int32_t)) == 0, "kSMEM must be a multiple of 8 bytes."); +// seq_lens[b] must not exceed scores.size(1) or page_table.size(1) << page_bits: a row reads up to its length struct TopKParams { const float* __restrict__ scores; const int32_t* __restrict__ seq_lens; @@ -57,6 +58,8 @@ struct TopKParams { uint32_t page_bits; uint32_t topk; int64_t output_stride; + // Emit each row's picks in ascending order (see bitonic_sort_u32). + bool sort_output; }; __device__ __forceinline__ uint8_t convert_to_uint8(float x) { @@ -251,6 +254,116 @@ radix_topk(const float* __restrict__ input, int32_t* __restrict__ output, uint32 } } +#ifdef USE_ROCM +// Bitonic sort of n (a power of two, 64 <= n <= kMaxTopK) 32-bit keys, one per thread: strides +// below the wavefront width exchange through lane shuffles, the wider ones through LDS. +#if defined(__GFX10__) || defined(__GFX11__) || defined(__GFX12__) +constexpr uint32_t kWaveSize = 32; // RDNA and gfx1250 run 32-lane waves +#else +constexpr uint32_t kWaveSize = 64; +#endif + +// lane ^ J's value in registers: DPP for J <= 8, gfx950 permlane swaps for J = 16, 32; __shfl_xor +// is an LDS round trip on every stage's dependent chain. +template +__device__ __forceinline__ uint32_t lane_xor(uint32_t v) { +#if defined(__gfx90a__) || defined(__gfx942__) || defined(__gfx950__) + if constexpr (J == 1) { // quad_perm [1, 0, 3, 2] + return static_cast(__builtin_amdgcn_update_dpp(0, static_cast(v), 0xB1, 0xF, 0xF, true)); + } + if constexpr (J == 2) { // quad_perm [2, 3, 0, 1] + return static_cast(__builtin_amdgcn_update_dpp(0, static_cast(v), 0x4E, 0xF, 0xF, true)); + } + if constexpr (J == 4 || J == 8) { + // within a 16-lane row: bit clear reads J lanes up (row_shl), bit set J lanes down (row_shr) + const uint32_t shl = + static_cast(__builtin_amdgcn_update_dpp(0, static_cast(v), 0x100 | J, 0xF, 0xF, true)); + const uint32_t shr = + static_cast(__builtin_amdgcn_update_dpp(0, static_cast(v), 0x110 | J, 0xF, 0xF, true)); + return (__lane_id() & J) ? shr : shl; + } +#endif +#if defined(__gfx950__) + if constexpr (J == 16) { + // rows 1 and 3 of the first operand swap with rows 0 and 2 of the second + const auto pair = __builtin_amdgcn_permlane16_swap(v, v, false, false); + return (__lane_id() & 16) ? pair[0] : pair[1]; + } + if constexpr (J == 32) { + const auto pair = __builtin_amdgcn_permlane32_swap(v, v, false, false); + return (__lane_id() & 32) ? pair[0] : pair[1]; + } +#endif + return static_cast(__shfl_xor(static_cast(v), static_cast(J), kWaveSize)); +} + +// One bitonic stage (merge size K, stride J); templated so the network unrolls with no stride switch. +template +__device__ __forceinline__ uint32_t bitonic_stage(uint32_t v, uint32_t* __restrict__ s_vals, uint32_t tx, uint32_t n) { + const bool up = (tx & K) == 0; + const bool lower = (tx & J) == 0; + uint32_t w; + if constexpr (J >= kWaveSize) { + __syncthreads(); + if (tx < n) s_vals[tx] = v; + __syncthreads(); + w = tx < n ? s_vals[tx ^ J] : ~0u; + } else { + w = lane_xor(v); + } + return (lower == up) ? min(v, w) : max(v, w); +} + +template +__device__ __forceinline__ uint32_t bitonic_network(uint32_t v, uint32_t* __restrict__ s_vals, uint32_t tx) { + v = bitonic_stage(v, s_vals, tx, N); + if constexpr (J > 1) { + return bitonic_network(v, s_vals, tx); + } else if constexpr (K < N) { + return bitonic_network(v, s_vals, tx); + } else { + return v; + } +} + +// Sort the N (a power of two, 64 <= N <= kBlockSize) values in s_vals ascending, one per thread. +template +__device__ void bitonic_sort_fixed(uint32_t* __restrict__ s_vals) { + static_assert(N >= 64 && N <= kBlockSize && (N & (N - 1)) == 0, "one value per thread"); + const uint32_t tx = threadIdx.x; + uint32_t v = tx < N ? s_vals[tx] : ~0u; + v = bitonic_network(v, s_vals, tx); + __syncthreads(); + if (tx < N) s_vals[tx] = v; + __syncthreads(); +} + +// one value per thread: the block covers the largest top-k +static_assert(kBlockSize >= kMaxTopK); + +__device__ void bitonic_sort_u32(uint32_t* __restrict__ s_vals, uint32_t n) { + switch (n) { + case 64: + return bitonic_sort_fixed<64>(s_vals); + case 128: + return bitonic_sort_fixed<128>(s_vals); + case 256: + return bitonic_sort_fixed<256>(s_vals); + case 512: + return bitonic_sort_fixed<512>(s_vals); + default: + return bitonic_sort_fixed(s_vals); + } +} + +__device__ __forceinline__ uint32_t next_pow2_at_least_64(uint32_t x) { + uint32_t n = 64; + while (n < x) + n <<= 1; + return n; +} +#endif // USE_ROCM + __global__ __launch_bounds__(kBlockSize) void deepseek_v4_topk_transform_kernel(const TopKParams params) { const auto bid = blockIdx.x; const auto seq_len = params.seq_lens[bid]; @@ -261,15 +374,63 @@ __global__ __launch_bounds__(kBlockSize) void deepseek_v4_topk_transform_kernel( const auto raw_indices_ptr = params.raw_indices != nullptr ? params.raw_indices + bid * params.output_stride : nullptr; + __shared__ int32_t s_topk_indices[kMaxTopK]; + + // key: the position when the row has raw indices, else the slot + const bool key_is_position = raw_indices_ptr != nullptr; + uint32_t count = topk; if (seq_len <= static_cast(topk)) { - naive_paged_transform(seq_len, topk, params.page_bits, page_ptr, indices_ptr, raw_indices_ptr); + if (!params.sort_output || key_is_position) { + // ascending positions with the -1 padding last: already the sorted row + naive_paged_transform(seq_len, topk, params.page_bits, page_ptr, indices_ptr, raw_indices_ptr); + return; + } + // every position is a pick; the slot order still has to be established + count = static_cast(seq_len); + for (uint32_t i = threadIdx.x; i < count; i += kBlockSize) { + s_topk_indices[i] = static_cast(i); + } + } else { + radix_topk(score_ptr, s_topk_indices, static_cast(seq_len), topk); + } + __syncthreads(); + +#ifdef USE_ROCM + if (params.sort_output) { + __shared__ uint32_t s_sort_vals[kMaxTopK]; + const uint32_t n = next_pow2_at_least_64(count); + for (uint32_t i = threadIdx.x; i < n; i += kBlockSize) { + uint32_t key = ~0u; + if (i < count) { + const int32_t raw = s_topk_indices[i]; + key = static_cast( + key_is_position ? raw : page_to_slot(page_ptr, static_cast(raw), params.page_bits)); + } + s_sort_vals[i] = key; + } + __syncthreads(); + bitonic_sort_u32(s_sort_vals, n); + for (uint32_t i = threadIdx.x; i < topk; i += kBlockSize) { + int32_t slot = -1; + int32_t raw = -1; + if (i < count) { + const int32_t key = static_cast(s_sort_vals[i]); + if (key_is_position) { + raw = key; + slot = page_to_slot(page_ptr, static_cast(raw), params.page_bits); + } else { + slot = key; + } + } + indices_ptr[i] = slot; + if (raw_indices_ptr != nullptr) { + raw_indices_ptr[i] = raw; + } + } return; } +#endif - __shared__ int32_t s_topk_indices[kMaxTopK]; - radix_topk(score_ptr, s_topk_indices, static_cast(seq_len), topk); - - __syncthreads(); for (uint32_t i = threadIdx.x; i < topk; i += kBlockSize) { const auto raw = s_topk_indices[i]; indices_ptr[i] = page_to_slot(page_ptr, static_cast(raw), params.page_bits); @@ -304,11 +465,15 @@ void deepseek_v4_topk_transform_512( const at::Tensor& page_table, at::Tensor& page_indices, int64_t page_size, - std::optional raw_indices_opt) { + std::optional raw_indices_opt, + bool sort_output) { CHECK_CUDA(scores); CHECK_CUDA(seq_lens); CHECK_CUDA(page_table); CHECK_CUDA(page_indices); +#ifndef USE_ROCM + TORCH_CHECK(!sort_output, "deepseek_v4_topk_transform_512: sort_output is only built for ROCm"); +#endif if (raw_indices_opt.has_value()) { CHECK_CUDA(raw_indices_opt.value()); } @@ -367,6 +532,7 @@ void deepseek_v4_topk_transform_512( .page_bits = page_bits, .topk = static_cast(topk), .output_stride = topk, + .sort_output = sort_output, }; const auto stream = at::cuda::getCurrentCUDAStream().stream(); diff --git a/python/sglang/kernels/aot/include/sgl_kernel_ops.h b/python/sglang/kernels/aot/include/sgl_kernel_ops.h index 9e3bb1dede..65c274b86c 100644 --- a/python/sglang/kernels/aot/include/sgl_kernel_ops.h +++ b/python/sglang/kernels/aot/include/sgl_kernel_ops.h @@ -170,7 +170,8 @@ void deepseek_v4_topk_transform_512( const at::Tensor& page_table, at::Tensor& page_indices, int64_t page_size, - std::optional raw_indices_opt = std::nullopt); + std::optional raw_indices_opt = std::nullopt, + bool sort_output = false); #endif /* diff --git a/python/sglang/kernels/aot/python/sgl_kernel/top_k.py b/python/sglang/kernels/aot/python/sgl_kernel/top_k.py index c93dcd05c8..6826ed1aee 100644 --- a/python/sglang/kernels/aot/python/sgl_kernel/top_k.py +++ b/python/sglang/kernels/aot/python/sgl_kernel/top_k.py @@ -87,6 +87,7 @@ def deepseek_v4_topk_transform_512( page_indices: torch.Tensor, page_size: int, raw_indices: Optional[torch.Tensor] = None, + sort_output: bool = False, ) -> None: """ Performs the DeepSeek-V4 indexer top-k selection and writes the paged @@ -96,7 +97,8 @@ def deepseek_v4_topk_transform_512( Args: scores: float32 ``[B, max_seq_len]`` indexer logits, contiguous on dim 1. - seq_lens: int32 ``[B]``, true KV length per batch row. + seq_lens: int32 ``[B]``, true KV length per batch row; each at most + ``max_seq_len`` and ``num_pages * page_size``. page_table: int32 ``[B, num_pages]``, logical->physical page table, contiguous on dim 1. page_indices: int32 ``[B, topk]``, output buffer, contiguous. Filled @@ -104,11 +106,14 @@ def deepseek_v4_topk_transform_512( page_size: power-of-2 page size. raw_indices: optional int32 ``[B, topk]``, contiguous. If provided, filled with raw token positions within each row. + sort_output: sort every row ascending in the kernel epilogue, with the + -1 padding last: by raw position when ``raw_indices`` is given, + otherwise by physical slot. The selected set is unchanged. """ if raw_indices is not None: assert raw_indices.dim() == 2 torch.ops.sgl_kernel.deepseek_v4_topk_transform_512( - scores, seq_lens, page_table, page_indices, page_size, raw_indices + scores, seq_lens, page_table, page_indices, page_size, raw_indices, sort_output ) diff --git a/python/sglang/kernels/aot/tests/test_topk.py b/python/sglang/kernels/aot/tests/test_topk.py index db788f1902..4de40917af 100644 --- a/python/sglang/kernels/aot/tests/test_topk.py +++ b/python/sglang/kernels/aot/tests/test_topk.py @@ -292,6 +292,51 @@ def test_deepseek_v4_topk_transform(bs: int, c4_len: int) -> None: ) +@pytest.mark.skipif( + torch.version.hip is None, + reason="deepseek_v4_topk_transform_512 is only built on ROCm", +) +@pytest.mark.parametrize("with_raw", [False, True]) +@torch.inference_mode() +def test_deepseek_v4_topk_transform_sorted_output(with_raw: bool) -> None: + """sort_output orders each row ascending (by raw position when given), -1 padding last, + without changing the selected set, also for rows shorter than topk.""" + from sgl_kernel import deepseek_v4_topk_transform_512 + + torch.manual_seed(7) + topk, page_size, width = 512, 64, 5000 + lens = [0, 1, topk - 1, topk, topk + 1, 3000, width] + bs = len(lens) + scores = torch.randn(bs, width, dtype=torch.float32, device="cuda") + seq_lens = torch.tensor(lens, dtype=torch.int32, device="cuda") + num_pages = (width + page_size - 1) // page_size + page_table = torch.stack( + [torch.randperm(4 * num_pages, device="cuda")[:num_pages] for _ in range(bs)] + ).int() + + outs = [] + for sort_output in (False, True): + page = torch.empty(bs, topk, dtype=torch.int32, device="cuda") + raw = torch.empty_like(page) if with_raw else None + deepseek_v4_topk_transform_512( + scores, seq_lens, page_table, page, page_size, raw, sort_output=sort_output + ) + outs.append((page, raw)) + (page, raw), (page_sorted, raw_sorted) = outs + for b in range(bs): + key = (raw if with_raw else page)[b] + chosen = key[key >= 0].sort().values + padding = torch.full( + (topk - chosen.numel(),), -1, dtype=torch.int32, device="cuda" + ) + assert torch.equal( + (raw_sorted if with_raw else page_sorted)[b], torch.cat([chosen, padding]) + ) + if with_raw: # each slot still travels with its raw position + pairs = sorted(zip(page[b].tolist(), raw[b].tolist())) + assert pairs == sorted(zip(page_sorted[b].tolist(), raw_sorted[b].tolist())) + + def _make_scores(kind: str, bs: int, width: int, seed: int) -> torch.Tensor: """Score distributions that stress the coarse stage of a histogram top-k. diff --git a/python/sglang/kernels/ops/attention/dsv4/attn_glue_hip.py b/python/sglang/kernels/ops/attention/dsv4/attn_glue_hip.py new file mode 100644 index 0000000000..c65da99fde --- /dev/null +++ b/python/sglang/kernels/ops/attention/dsv4/attn_glue_hip.py @@ -0,0 +1,426 @@ +"""Single-launch replacements for the torch glue around the DeepSeek-V4.1 sparse attention metadata +on HIP; every kernel reproduces the torch expression quoted in its docstring bit for bit.""" + +from __future__ import annotations + +from typing import Dict, Optional, Tuple + +import torch +import triton +import triton.language as tl + + +@triton.jit +def _mask_indices_by_length_kernel( + idx_ptr, + len_ptr, + out_ptr, + idx2_ptr, + len2_ptr, + out2_ptr, + rows_per_len, + W1: tl.constexpr, + W2: tl.constexpr, + BLOCK: tl.constexpr, + HAS_SECOND: tl.constexpr, +): + row = tl.program_id(0) + which = tl.program_id(1) + length_row = row // rows_per_len + offs = tl.arange(0, BLOCK) + if which == 0: + length = tl.load(len_ptr + length_row) + for w0 in tl.static_range(0, W1, BLOCK): + w = w0 + offs + m = w < W1 + v = tl.load(idx_ptr + row * W1 + w, mask=m, other=0) + tl.store(out_ptr + row * W1 + w, tl.where(w < length, v, -1), mask=m) + elif HAS_SECOND: + length = tl.load(len2_ptr + length_row) + for w0 in tl.static_range(0, W2, BLOCK): + w = w0 + offs + m = w < W2 + v = tl.load(idx2_ptr + row * W2 + w, mask=m, other=0) + tl.store(out2_ptr + row * W2 + w, tl.where(w < length, v, -1), mask=m) + + +def _mask_shape(indices: torch.Tensor, lengths: torch.Tensor) -> Tuple[int, int, int]: + assert indices.is_contiguous(), indices.stride() + assert lengths.dim() == 1 and lengths.is_contiguous() + w = indices.shape[-1] + rows = indices.numel() // w if w else 0 + batch = lengths.shape[0] + assert batch > 0 and rows % batch == 0, (indices.shape, lengths.shape) + return rows, rows // batch, w + + +def mask_indices_by_length( + indices: torch.Tensor, + lengths: torch.Tensor, + indices2: Optional[torch.Tensor] = None, + lengths2: Optional[torch.Tensor] = None, +) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: + """torch.where(arange(w) < lengths.view(-1, 1, 1), indices, -1) for one or + two [b, s, w] index tensors in one launch (the sparse decode kernel takes + no per-row length and skips -1). lengths is [b]; every s row of a + batch entry shares its length. Returns new tensors of the input dtypes.""" + rows, rows_per_len, w1 = _mask_shape(indices, lengths) + out = torch.empty_like(indices) + has_second = indices2 is not None + if has_second: + rows2, rows_per_len2, w2 = _mask_shape(indices2, lengths2) + assert rows2 == rows and rows_per_len2 == rows_per_len, ( + indices.shape, + indices2.shape, + ) + out2 = torch.empty_like(indices2) + else: + w2 = w1 + indices2, lengths2, out2 = indices, lengths, out + if rows == 0: + return out, (out2 if has_second else None) + block = min(1024, triton.next_power_of_2(max(w1, w2))) + _mask_indices_by_length_kernel[(rows, 2 if has_second else 1)]( + indices, + lengths, + out, + indices2, + lengths2, + out2, + rows_per_len, + W1=w1, + W2=w2, + BLOCK=block, + HAS_SECOND=has_second, + num_warps=4, + ) + return out, (out2 if has_second else None) + + +@triton.jit +def _expand_index_page_table_kernel( + pt_ptr, + out_ptr, + n_pages, + pt_stride, + BPP: tl.constexpr, + BLOCK: tl.constexpr, +): + row = tl.program_id(0) + p = tl.program_id(1) * BLOCK + tl.arange(0, BLOCK) + m = p < n_pages + page = tl.load(pt_ptr + row * pt_stride + p, mask=m, other=0) + base = page.to(tl.int64) * BPP + k = tl.arange(0, BPP) + expanded = base[:, None] + k[None, :] + tl.store( + out_ptr + row * (n_pages * BPP) + p[:, None] * BPP + k[None, :], + expanded.to(tl.int32), + mask=m[:, None], + ) + + +def expand_index_page_table( + page_table: torch.Tensor, blocks_per_page: int +) -> torch.Tensor: + """(page_table.to(int64) * bpp).unsqueeze(-1) + arange(bpp) reshaped to + [bs, n * bpp] int32: the block table of a low-ratio indexer-K pool that + pages at a fraction of a FULL page (candidate_indexer.expand_index_page_table).""" + if blocks_per_page == 1: + return page_table + assert blocks_per_page & (blocks_per_page - 1) == 0, blocks_per_page + assert page_table.dim() == 2 and page_table.stride(1) == 1 + bs, n = page_table.shape + out = torch.empty( + (bs, n * blocks_per_page), dtype=torch.int32, device=page_table.device + ) + if bs == 0 or n == 0: + return out + block = 256 + _expand_index_page_table_kernel[(bs, triton.cdiv(n, block))]( + page_table, + out, + n, + page_table.stride(0), + BPP=blocks_per_page, + BLOCK=block, + num_warps=4, + ) + return out + + +@triton.jit +def _low_ratio_compression_metadata_kernel( + seq_lens_ptr, + raw_out_loc_ptr, + c1_out_loc_ptr, + c1_clamp1_ptr, + c2_out_loc_ptr, + c2_clamp1_ptr, + rows, + num_write_tokens, + HAS_C1: tl.constexpr, + HAS_C2: tl.constexpr, + BLOCK: tl.constexpr, +): + r = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK) + m = r < rows + mw = r < num_write_tokens + seq_len = tl.load(seq_lens_ptr + r, mask=m, other=0) + raw_out_loc = tl.load(raw_out_loc_ptr + r, mask=mw, other=0).to(tl.int64) + if HAS_C1: + # ratio 1: every token completes a group, so out_loc is raw_out_loc itself + tl.store(c1_out_loc_ptr + r, raw_out_loc, mask=mw) + tl.store(c1_clamp1_ptr + r, tl.maximum(seq_len, 1), mask=m) + if HAS_C2: + completes = (seq_len % 2) == 0 + # >> 1 floors as torch's // does; Triton's // rounds toward zero on a negative loc + tl.store(c2_out_loc_ptr + r, tl.where(completes, raw_out_loc >> 1, -1), mask=mw) + tl.store(c2_clamp1_ptr + r, tl.maximum(seq_len // 2, 1), mask=m) + + +def low_ratio_compression_metadata( + seq_lens_casual: torch.Tensor, + raw_out_loc: torch.Tensor, + low_ratios: Tuple[int, ...], +) -> Dict[str, torch.Tensor]: + """_low_ratio_compression_metadata for ratios 1 and 2 in one launch: cR_out_loc = + where(seq_lens[:nw] % R == 0, raw_out_loc // R, -1) (int64) and cR_topk_lengths_clamp1 = + (seq_lens // R).clamp_min(1) (int32), keyed c{R}_... for the ratios present.""" + assert seq_lens_casual.dtype is torch.int32 and seq_lens_casual.is_contiguous() + assert raw_out_loc.dim() == 1 and raw_out_loc.is_contiguous() + assert set(low_ratios) <= {1, 2}, low_ratios + rows = seq_lens_casual.shape[0] + nw = raw_out_loc.shape[0] + assert nw <= rows, (nw, rows) + device = seq_lens_casual.device + out = {} + for ratio in (1, 2): + if ratio in low_ratios: + out[f"c{ratio}_out_loc"] = torch.empty(nw, dtype=torch.int64, device=device) + out[f"c{ratio}_topk_lengths_clamp1"] = torch.empty( + rows, dtype=torch.int32, device=device + ) + if not out or rows == 0: + return out + placeholder = seq_lens_casual + _low_ratio_compression_metadata_kernel[(triton.cdiv(rows, 1024),)]( + seq_lens_casual, + raw_out_loc, + out.get("c1_out_loc", placeholder), + out.get("c1_topk_lengths_clamp1", placeholder), + out.get("c2_out_loc", placeholder), + out.get("c2_topk_lengths_clamp1", placeholder), + rows, + nw, + HAS_C1=1 in low_ratios, + HAS_C2=2 in low_ratios, + BLOCK=1024, + num_warps=4, + ) + return out + + +@triton.jit +def _sparse_buffers_kernel( + c4_clamp1_ptr, + c4_raw_ptr, + c4_sparse_ptr, + c4_sparse_raw_ptr, + c4_page_ptr, + c1_clamp1_ptr, + c1_sparse_ptr, + c1_page_ptr, + c2_clamp1_ptr, + c2_sparse_ptr, + c2_page_ptr, + rows, + topk, + width, + HAS_C1: tl.constexpr, + HAS_C2: tl.constexpr, + BLOCK: tl.constexpr, + FILL_BLOCK: tl.constexpr, +): + pid = tl.program_id(0) + if pid == 0: + # one program clamps the per-row lengths: a few ints per row + for r0 in range(0, rows, BLOCK): + r = r0 + tl.arange(0, BLOCK) + m = r < rows + c4_clamp1 = tl.load(c4_clamp1_ptr + r, mask=m, other=0) + c4_raw = tl.load(c4_raw_ptr + r, mask=m, other=0) + tl.store(c4_sparse_ptr + r, tl.minimum(c4_clamp1, topk), mask=m) + tl.store(c4_sparse_raw_ptr + r, tl.minimum(c4_raw, topk), mask=m) + if HAS_C1: + c1_clamp1 = tl.load(c1_clamp1_ptr + r, mask=m, other=0) + tl.store(c1_sparse_ptr + r, tl.minimum(c1_clamp1, topk), mask=m) + if HAS_C2: + c2_clamp1 = tl.load(c2_clamp1_ptr + r, mask=m, other=0) + tl.store(c2_sparse_ptr + r, tl.minimum(c2_clamp1, topk), mask=m) + else: + # the rest fill the -1 page-index buffers, one buffer per range of programs + fill = pid - 1 + n = rows * width + per_buffer = tl.cdiv(n, FILL_BLOCK) + which = fill // per_buffer + i = (fill - which * per_buffer) * FILL_BLOCK + tl.arange(0, FILL_BLOCK) + m = i < n + if which == 0: + tl.store(c4_page_ptr + i, -1, mask=m) + elif which == 1: + if HAS_C1: + tl.store(c1_page_ptr + i, -1, mask=m) + elif HAS_C2: + tl.store(c2_page_ptr + i, -1, mask=m) + else: + if HAS_C1 and HAS_C2: + tl.store(c2_page_ptr + i, -1, mask=m) + + +def sparse_buffers( + *, + c4_topk_lengths_clamp1: torch.Tensor, + c4_topk_lengths_raw: torch.Tensor, + c1_topk_lengths_clamp1: Optional[torch.Tensor], + c2_topk_lengths_clamp1: Optional[torch.Tensor], + index_topk: int, + page_index_align: int, +) -> Dict[str, torch.Tensor]: + """init_flashmla_related's tensors in one launch, keyed by attribute name: the top-k + lengths clamped to index_topk and the -1-filled int32 page-index buffers padded to + page_index_align, for ratio 4 and the ratios whose clamp-1 lengths are given.""" + for t in ( + c4_topk_lengths_clamp1, + c4_topk_lengths_raw, + c1_topk_lengths_clamp1, + c2_topk_lengths_clamp1, + ): + assert t is None or ( + t.dtype is torch.int32 and t.dim() == 1 and t.is_contiguous() + ), t + rows = c4_topk_lengths_clamp1.shape[0] + assert c4_topk_lengths_raw.shape[0] == rows + device = c4_topk_lengths_clamp1.device + width = triton.cdiv(index_topk, page_index_align) * page_index_align + i32 = dict(dtype=torch.int32, device=device) + out = { + "c4_sparse_topk_lengths": torch.empty(rows, **i32), + "c4_sparse_topk_lengths_raw": torch.empty(rows, **i32), + "c4_sparse_page_indices": torch.empty((rows, width), **i32), + } + has_c1 = c1_topk_lengths_clamp1 is not None + has_c2 = c2_topk_lengths_clamp1 is not None + for ratio, has in ((1, has_c1), (2, has_c2)): + if has: + out[f"c{ratio}_sparse_topk_lengths"] = torch.empty(rows, **i32) + out[f"c{ratio}_sparse_page_indices"] = torch.empty((rows, width), **i32) + if rows == 0: + return out + placeholder = out["c4_sparse_topk_lengths"] + fill_block = 1024 + per_buffer = triton.cdiv(rows * width, fill_block) + n_buffers = 1 + int(has_c1) + int(has_c2) + _sparse_buffers_kernel[(1 + per_buffer * n_buffers,)]( + c4_topk_lengths_clamp1, + c4_topk_lengths_raw, + out["c4_sparse_topk_lengths"], + out["c4_sparse_topk_lengths_raw"], + out["c4_sparse_page_indices"], + c1_topk_lengths_clamp1 if has_c1 else placeholder, + out.get("c1_sparse_topk_lengths", placeholder), + out.get("c1_sparse_page_indices", placeholder), + c2_topk_lengths_clamp1 if has_c2 else placeholder, + out.get("c2_sparse_topk_lengths", placeholder), + out.get("c2_sparse_page_indices", placeholder), + rows, + index_topk, + width, + HAS_C1=has_c1, + HAS_C2=has_c2, + BLOCK=1024, + FILL_BLOCK=fill_block, + num_warps=4, + ) + return out + + +@triton.jit +def _page_table_from_req_to_token_kernel( + r2t_ptr, + req_ptr, + out_ptr, + num_cols, + r2t_stride, + PAGE: tl.constexpr, + BLOCK: tl.constexpr, +): + row = tl.program_id(0) + blk = tl.program_id(1) + req = tl.load(req_ptr + row).to(tl.int64) + col = blk * BLOCK + tl.arange(0, BLOCK) + mask = col < num_cols + v = tl.load( + r2t_ptr + req * r2t_stride + col.to(tl.int64) * PAGE, mask=mask, other=0 + ) + # torch's floor division; the C-style quotient rounds toward zero + q = v // PAGE + q = tl.where((v < 0) & (v % PAGE != 0), q - 1, q) + tl.store(out_ptr + row.to(tl.int64) * num_cols + col, q.to(tl.int32), mask=mask) + + +def page_table_from_req_to_token( + req_to_token: torch.Tensor, + req_rows: torch.Tensor, + max_seq_len: int, + page_size: int, +) -> torch.Tensor: + """(req_to_token[req_rows, :max_seq_len:page_size] // page_size).to(torch.int32) in one + launch (the gather, the floor division and the cast).""" + assert req_to_token.dim() == 2 and req_to_token.stride(1) == 1 + assert req_rows.dim() == 1 and req_rows.stride(0) == 1 + n = req_rows.shape[0] + num_cols = max( + 0, (min(max_seq_len, req_to_token.shape[1]) + page_size - 1) // page_size + ) + out = torch.empty((n, num_cols), dtype=torch.int32, device=req_to_token.device) + if n == 0 or num_cols == 0: + return out + block = min(1024, triton.next_power_of_2(num_cols)) + _page_table_from_req_to_token_kernel[(n, triton.cdiv(num_cols, block))]( + req_to_token, + req_rows, + out, + num_cols, + req_to_token.stride(0), + PAGE=page_size, + BLOCK=block, + num_warps=4, + ) + return out + + +@triton.jit +def _widen_pair_kernel(a_ptr, b_ptr, oa_ptr, ob_ptr, n, BLOCK: tl.constexpr): + i = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK) + mask = i < n + tl.store(oa_ptr + i, tl.load(a_ptr + i, mask=mask, other=0).to(tl.int64), mask=mask) + tl.store(ob_ptr + i, tl.load(b_ptr + i, mask=mask, other=0).to(tl.int64), mask=mask) + + +def widen_pair_i64( + a: torch.Tensor, b: torch.Tensor +) -> Tuple[torch.Tensor, torch.Tensor]: + """(a.to(torch.int64), b.to(torch.int64)) for two same-length integer vectors in one + launch (the decode request slots and positions the low-ratio indexer takes widened).""" + assert a.dim() == b.dim() == 1 and a.shape == b.shape, (a.shape, b.shape) + assert a.stride(0) == 1 and b.stride(0) == 1 + n = a.shape[0] + oa = torch.empty(n, dtype=torch.int64, device=a.device) + ob = torch.empty(n, dtype=torch.int64, device=a.device) + if n: + block = min(1024, triton.next_power_of_2(n)) + _widen_pair_kernel[(triton.cdiv(n, block),)]( + a, b, oa, ob, n, BLOCK=block, num_warps=4 + ) + return oa, ob diff --git a/python/sglang/kernels/ops/attention/dsv4/candidate_blocks_hip.py b/python/sglang/kernels/ops/attention/dsv4/candidate_blocks_hip.py new file mode 100644 index 0000000000..a48fb1615b --- /dev/null +++ b/python/sglang/kernels/ops/attention/dsv4/candidate_blocks_hip.py @@ -0,0 +1,408 @@ +"""Candidate blocks of the DeepSeek V4.1 two-level indexer on ROCm: level-one block scores and +top blocks, the top-k within them, and the sorted paged top-k (the HIP side of candidate_blocks).""" + +from __future__ import annotations + +from typing import List, NamedTuple, Optional + +import torch +import torch.nn.functional as F +import triton +import triton.language as tl + +from sglang.kernels.ops.attention.dsv4 import topk_transform_paged + +# the k the AOT fast_topk op (topk_hip.hip) is instantiated for; any other k takes torch.topk +_AOT_FAST_TOPK_K = 2048 + + +def topk_transform_paged_sorted( + scores: torch.Tensor, + seq_lens: torch.Tensor, + page_table: torch.Tensor, + page_indices: torch.Tensor, + page_size: int, + raw_indices: Optional[torch.Tensor], +) -> None: + """topk_transform_paged with every row sorted ascending by position in the AOT + kernel's epilogue, -1 padding last.""" + torch.ops.sgl_kernel.deepseek_v4_topk_transform_512( + scores, seq_lens, page_table, page_indices, page_size, raw_indices, True + ) + + +# Candidate blocks one Triton program reduces; times block_size positions of logits. +_LEVEL_ONE_BLOCKS_PER_PROGRAM = 256 + + +@triton.jit +def _candidate_block_scores_kernel( + logits_ptr, + seq_lens_ptr, + out_ptr, + logits_stride, + width, + num_blocks, + out_stride, + BLOCK_SIZE: tl.constexpr, + BLOCKS_PER_PROGRAM: tl.constexpr, + FILL_TAIL: tl.constexpr, +): + """out[row, blk] = max(logits[row, blk * BLOCK_SIZE : (blk + 1) * BLOCK_SIZE]) + over the positions < seq_lens[row]; +inf for the block holding the newest + position, -inf for blocks past the reach (written only with FILL_TAIL). A + program whose blocks all lie past the reach reads no logits.""" + row = tl.program_id(0) + block0 = tl.program_id(1) * BLOCKS_PER_PROGRAM + length = tl.load(seq_lens_ptr + row) + blocks = block0 + tl.arange(0, BLOCKS_PER_PROGRAM) + in_table = blocks < num_blocks + if block0 * BLOCK_SIZE >= length: + if FILL_TAIL: + tl.store(out_ptr + row * out_stride + blocks, float("-inf"), mask=in_table) + return + cols = blocks[:, None] * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)[None, :] + vals = tl.load( + logits_ptr + row * logits_stride + cols, + mask=(cols < length) & (cols < width), + other=float("-inf"), + ) + scores = tl.max(vals, axis=1) + last = (length - 1) // BLOCK_SIZE + scores = tl.where(blocks == last, float("inf"), scores) + tl.store(out_ptr + row * out_stride + blocks, scores, mask=in_table) + + +@triton.jit +def _gather_candidate_blocks_kernel( + logits_ptr, + seq_lens_ptr, + ids_ptr, + out_ptr, + logits_stride, + width, + ids_stride, + out_stride, + BLOCK_SIZE: tl.constexpr, + BLOCKS_PER_PROGRAM: tl.constexpr, +): + """out[row, j * BLOCK_SIZE + t] = logits[row, ids[row, j] * BLOCK_SIZE + t] for + the reachable positions of the kept blocks, -inf elsewhere (an id of -1, or a + position past the reach in the newest block).""" + row = tl.program_id(0) + j0 = tl.program_id(1) * BLOCKS_PER_PROGRAM + length = tl.load(seq_lens_ptr + row) + j = j0 + tl.arange(0, BLOCKS_PER_PROGRAM) + ids = tl.load(ids_ptr + row * ids_stride + j) + t = tl.arange(0, BLOCK_SIZE)[None, :] + cols = ids[:, None] * BLOCK_SIZE + t + vals = tl.load( + logits_ptr + row * logits_stride + cols, + mask=(ids[:, None] >= 0) & (cols < length) & (cols < width), + other=float("-inf"), + ) + tl.store(out_ptr + row * out_stride + j[:, None] * BLOCK_SIZE + t, vals) + + +def candidate_block_scores( + logits: torch.Tensor, + seq_lens: torch.Tensor, + *, + block_size: int, + fill_tail: bool, +) -> torch.Tensor: + """[rows, num_blocks] fp32 block maxima of logits over the reachable positions + (see _candidate_block_scores_kernel). seq_lens int32 [rows], contiguous.""" + assert logits.dim() == 2 and logits.dtype == torch.float32 and logits.stride(1) == 1 + assert block_size & (block_size - 1) == 0, f"{block_size = } must be a power of 2" + rows, width = logits.shape + num_blocks = triton.cdiv(width, block_size) + scores = torch.empty((rows, num_blocks), dtype=torch.float32, device=logits.device) + grid = (rows, triton.cdiv(num_blocks, _LEVEL_ONE_BLOCKS_PER_PROGRAM)) + _candidate_block_scores_kernel[grid]( + logits, + seq_lens, + scores, + logits.stride(0), + width, + num_blocks, + scores.stride(0), + BLOCK_SIZE=block_size, + BLOCKS_PER_PROGRAM=_LEVEL_ONE_BLOCKS_PER_PROGRAM, + FILL_TAIL=fill_tail, + ) + return scores + + +@triton.jit +def _candidate_lengths_kernel( + seq_lens_ptr, + block_lens_ptr, + compact_lens_ptr, + rows, + topk_blocks, + BLOCK_SIZE: tl.constexpr, + BLOCK: tl.constexpr, +): + """block_lens = ceil(len / BLOCK_SIZE): the blocks with a reachable position; + compact_lens = min(block_lens, topk_blocks) * BLOCK_SIZE: the width of the + compact candidate row (every block is kept while there are at most + topk_blocks of them).""" + r = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK) + mask = r < rows + length = tl.load(seq_lens_ptr + r, mask=mask, other=0) + block_lens = (length + (BLOCK_SIZE - 1)) // BLOCK_SIZE + tl.store(block_lens_ptr + r, block_lens, mask=mask) + tl.store( + compact_lens_ptr + r, + tl.minimum(block_lens, topk_blocks) * BLOCK_SIZE, + mask=mask, + ) + + +@triton.jit +def _map_compact_selection_kernel( + compact_pos_ptr, + ids_ptr, + seq_lens_ptr, + page_table_ptr, + page_indices_ptr, + raw_indices_ptr, + ids_stride, + pt_stride, + out_stride, + n_pages, + BLOCK_SIZE: tl.constexpr, + PAGE_SIZE: tl.constexpr, + TOPK: tl.constexpr, + BLOCK: tl.constexpr, + WRITE_RAW: tl.constexpr, + SORT: tl.constexpr, + PAD_KEY: tl.constexpr, +): + """Compact position c -> real position ids[c // BLOCK_SIZE] * BLOCK_SIZE + c % BLOCK_SIZE -> + slot via the row's page table; reachable selections packed first, -1 after. With SORT the valid + prefix is ascending by (position, slot).""" + row = tl.program_id(0) + length = tl.load(seq_lens_ptr + row) + j = tl.arange(0, BLOCK) + in_k = j < TOPK + c = tl.load(compact_pos_ptr + row * TOPK + j, mask=in_k, other=-1) + valid = c >= 0 + cc = tl.where(valid, c, 0) + blk = tl.load(ids_ptr + row * ids_stride + cc // BLOCK_SIZE) + real = blk * BLOCK_SIZE + cc % BLOCK_SIZE + valid = valid & (blk >= 0) & (real < length) + page = tl.load( + page_table_ptr + row * pt_stride + tl.minimum(real // PAGE_SIZE, n_pages - 1), + mask=valid, + other=0, + ) + slot = page * PAGE_SIZE + real % PAGE_SIZE + if SORT: + # BLOCK == TOPK (power of two): the sort key is the position with raw indices, else the slot + if WRITE_RAW: + hi = real + else: + hi = slot + key = tl.where(valid, hi, PAD_KEY).to(tl.int64) << 32 + key = tl.sort( + key | (tl.where(valid, slot, -1).to(tl.int64) & 0xFFFFFFFF), dim=0 + ) + pad = (key >> 32) == PAD_KEY + tl.store( + page_indices_ptr + row * out_stride + j, + tl.where(pad, -1, (key & 0xFFFFFFFF).to(tl.int32)), + ) + if WRITE_RAW: + tl.store( + raw_indices_ptr + row * out_stride + j, + tl.where(pad, -1, (key >> 32).to(tl.int32)), + ) + else: + v = valid.to(tl.int32) + count = tl.sum(v, axis=0) + wpos = tl.cumsum(v, axis=0) - 1 + # The packed entries and the padding never share an address. + tl.store(page_indices_ptr + row * out_stride + j, -1, mask=in_k & (j >= count)) + tl.store(page_indices_ptr + row * out_stride + wpos, slot, mask=valid) + if WRITE_RAW: + tl.store( + raw_indices_ptr + row * out_stride + j, -1, mask=in_k & (j >= count) + ) + tl.store(raw_indices_ptr + row * out_stride + wpos, real, mask=valid) + + +class CandidateBlocks(NamedTuple): + """What the candidate-source layer publishes for the decode rows of a step.""" + + # int32 [rows, topk_blocks]: kept block ids in no particular order, -1 padded after the last + ids: torch.Tensor + # int32 [rows]: kept blocks * block_size, the width of the compact row. + compact_lens: torch.Tensor + # int32 [rows, 1] zeros; with page_size = compact_page_size a position maps to itself + compact_page_table: torch.Tensor + compact_page_size: int + block_size: int + + +def slice_candidate_blocks(candidates: CandidateBlocks, rows: slice) -> CandidateBlocks: + """The rows rows of a per-request publication.""" + return candidates._replace( + ids=candidates.ids[rows], + compact_lens=candidates.compact_lens[rows], + compact_page_table=candidates.compact_page_table[rows], + ) + + +def cat_candidate_blocks(pieces: List[CandidateBlocks]) -> CandidateBlocks: + """Row chunks of one request's publication, in row order.""" + if len(pieces) == 1: + return pieces[0] + return pieces[0]._replace( + ids=torch.cat([p.ids for p in pieces]), + compact_lens=torch.cat([p.compact_lens for p in pieces]), + compact_page_table=torch.cat([p.compact_page_table for p in pieces]), + ) + + +def select_candidate_blocks_hip( + logits: torch.Tensor, + seq_lens: torch.Tensor, + *, + topk_blocks: int, + block_size: int, +) -> CandidateBlocks: + """Level one over a [rows, capacity] logits rectangle, bounded by seq_lens: the topk_blocks + best blocks of each row, the block with its newest position always among them; as a set per + row the ids equal the reference's select_candidate_blocks. Graph-safe: no host sync.""" + seq_lens = seq_lens.to(torch.int32).contiguous() + rows, width = logits.shape + device = logits.device + num_blocks = triton.cdiv(width, block_size) + use_aot = topk_blocks == _AOT_FAST_TOPK_K + scores = candidate_block_scores( + logits, seq_lens, block_size=block_size, fill_tail=not use_aot + ) + block_lens = torch.empty(rows, dtype=torch.int32, device=device) + compact_lens = torch.empty(rows, dtype=torch.int32, device=device) + _candidate_lengths_kernel[(triton.cdiv(rows, 1024),)]( + seq_lens, + block_lens, + compact_lens, + rows, + topk_blocks, + BLOCK_SIZE=block_size, + BLOCK=1024, + ) + if use_aot: + # exact top-k over the first ceil(len / block_size) block scores of each row, -1 padded + ids = torch.empty((rows, topk_blocks), dtype=torch.int32, device=device) + torch.ops.sgl_kernel.fast_topk(scores, ids, block_lens, None) + else: + picked = scores.topk(min(topk_blocks, num_blocks), dim=-1) + ids = picked.indices.to(torch.int32).masked_fill( + picked.values == -torch.inf, -1 + ) + # the gather takes the width from ids and tiles it in 256-block programs + ids = F.pad(ids, (0, topk_blocks - ids.shape[1]), value=-1) + compact_width = topk_blocks * block_size + return CandidateBlocks( + ids=ids, + compact_lens=compact_lens, + compact_page_table=torch.zeros((rows, 1), dtype=torch.int32, device=device), + compact_page_size=triton.next_power_of_2(compact_width), + block_size=block_size, + ) + + +def gather_candidate_blocks( + logits: torch.Tensor, + seq_lens: torch.Tensor, + ids: torch.Tensor, + *, + block_size: int, +) -> torch.Tensor: + """The compact [rows, topk_blocks * block_size] fp32 row of each request's + candidate positions in id order (see _gather_candidate_blocks_kernel).""" + rows, width = logits.shape + topk_blocks = ids.shape[1] + assert ids.shape[0] == rows and ids.dtype == torch.int32 and ids.stride(1) == 1 + per_program = min(topk_blocks, _LEVEL_ONE_BLOCKS_PER_PROGRAM) + assert topk_blocks % per_program == 0, topk_blocks + compact = torch.empty( + (rows, topk_blocks * block_size), dtype=torch.float32, device=logits.device + ) + grid = (rows, topk_blocks // per_program) + _gather_candidate_blocks_kernel[grid]( + logits, + seq_lens.to(torch.int32).contiguous(), + ids, + compact, + logits.stride(0), + width, + ids.stride(0), + compact.stride(0), + BLOCK_SIZE=block_size, + BLOCKS_PER_PROGRAM=per_program, + ) + return compact + + +def topk_within_candidate_blocks_hip( + logits: torch.Tensor, + seq_lens: torch.Tensor, + candidates: CandidateBlocks, + *, + page_table: torch.Tensor, + page_size: int, + page_indices: torch.Tensor, + raw_indices: Optional[torch.Tensor], + sort_output: bool = False, +) -> None: + """Level two for a consumer layer: the top-k of logits inside the published candidate + blocks, written as the paged transform writes it (-1 padded, valid prefix first; ascending with + sort_output, k a power of two). Runs on the compact row, so the cost stops growing with context.""" + rows, width = logits.shape + topk = page_indices.shape[1] + block_size = candidates.block_size + seq_lens = seq_lens.to(torch.int32).contiguous() + compact = gather_candidate_blocks( + logits, seq_lens, candidates.ids, block_size=block_size + ) + assert compact.shape[1] <= candidates.compact_page_size + compact_pos = torch.empty((rows, topk), dtype=torch.int32, device=logits.device) + topk_transform_paged( + compact, + candidates.compact_lens, + candidates.compact_page_table, + compact_pos, + candidates.compact_page_size, + None, + ) + assert page_indices.stride(1) == 1 and page_indices.shape == (rows, topk) + assert page_table.stride(1) == 1 and page_table.shape[0] == rows + write_raw = raw_indices is not None + if write_raw: + assert raw_indices.shape == (rows, topk) and raw_indices.stride(1) == 1 + assert raw_indices.stride(0) == page_indices.stride(0) + assert not sort_output or topk & (topk - 1) == 0, topk + _map_compact_selection_kernel[(rows,)]( + compact_pos, + candidates.ids, + seq_lens, + page_table, + page_indices, + raw_indices if write_raw else page_indices, + candidates.ids.stride(0), + page_table.stride(0), + page_indices.stride(0), + page_table.shape[1], + BLOCK_SIZE=block_size, + PAGE_SIZE=page_size, + TOPK=topk, + BLOCK=triton.next_power_of_2(topk), + WRITE_RAW=write_raw, + SORT=sort_output, + PAD_KEY=torch.iinfo(torch.int32).max, + ) diff --git a/python/sglang/kernels/ops/attention/dsv4/compact_attention_hip.py b/python/sglang/kernels/ops/attention/dsv4/compact_attention_hip.py new file mode 100644 index 0000000000..5d0ad598e8 --- /dev/null +++ b/python/sglang/kernels/ops/attention/dsv4/compact_attention_hip.py @@ -0,0 +1,403 @@ +"""Sparse attention reading V4.1 FP8/FP4 pages directly on HIP.""" + +from typing import Optional, Tuple + +import torch +import triton +from aiter.ops.triton._gluon_kernels.gfx950.attention.pa_decode_sparse import _qkpv_fp8 +from triton.experimental import gluon +from triton.experimental.gluon import language as gl + +from sglang.kernels.ops.attention.dsv4.kv_layout import KVLayout +from sglang.kernels.ops.attention.sparse_decode_reduce_hip import ( + gfx_sparse_split_reduce, +) + +RCP_LN2 = gl.constexpr(1.4426950408889634) # exp(x) = exp2(x * RCP_LN2) +TILE_SLOTS = 64 +MAX_SPLITS = 8 +PACKED_TOKEN_BYTES = (KVLayout.V41.bytes_per_token, KVLayout.V41_FP4.bytes_per_token) + + +@gluon.jit +def _load_compact(Cache, offset, valid, USE_BUFFER: gl.constexpr): + if USE_BUFFER: + return gl.amd.cdna4.buffer_load(Cache, offset.to(gl.int32), valid, other=0) + return gl.load(Cache + offset, valid, 0) + + +@gluon.jit +def _read_compact_gluon( + Cache, + slots, + valid, + PAGE: gl.constexpr, + STRIDE: gl.constexpr, + FP4: gl.constexpr, + USE_BUFFER: gl.constexpr, + G: gl.constexpr, +): + d = gl.arange(0, 512, layout=gl.SliceLayout(0, G)) + base = (slots // PAGE).to(gl.int64) * STRIDE + row = slots % PAGE + if FP4: + PG: gl.constexpr = gl.BlockedLayout([1, 8], [8, 8], [1, 4], [1, 0]) + pd = gl.arange(0, 256, layout=gl.SliceLayout(0, PG)) + packed_base = gl.convert_layout(base, gl.SliceLayout(1, PG)) + packed_row = gl.convert_layout(row, gl.SliceLayout(1, PG)) + packed_valid = gl.convert_layout(valid, gl.SliceLayout(1, PG)) + packed = _load_compact( + Cache, + packed_base[:, None] + packed_row[:, None] * 256 + pd[None, :], + packed_valid[:, None], + USE_BUFFER, + ) + value = gl.amd.cdna4.scaled_upcast( + packed, + gl.full((slots.shape[0], 512), 127, gl.uint8, G), + gl.bfloat16, + axis=1, + ).to(gl.float32) + scale = ( + _load_compact( + Cache, + base[:, None] + PAGE * 256 + row[:, None] * 32 + d[None, :] // 16, + valid[:, None], + USE_BUFFER, + ) + .to(gl.float8e4nv, bitcast=True) + .to(gl.float32) + ) + else: + value = ( + _load_compact( + Cache, + base[:, None] + row[:, None] * 512 + d[None, :], + valid[:, None], + USE_BUFFER, + ) + .to(gl.float8e4nv, bitcast=True) + .to(gl.float32) + ) + exp = _load_compact( + Cache, + base[:, None] + PAGE * 512 + row[:, None] * 16 + d[None, :] // 32, + valid[:, None], + USE_BUFFER, + ).to(gl.int32) + scale = gl.where(exp == 0, 0x00400000, exp << 23).to(gl.float32, bitcast=True) + scale = gl.where(exp == 255, float("nan"), scale) + return (value * scale).to(gl.bfloat16) + + +@gluon.jit +def _compact_qkpv( + kv, + valid, + qdot, + m, + den, + acc, + head_mask, + smem, + MF: gl.constexpr, + H: gl.constexpr, + BLOCK: gl.constexpr, + SCALE: gl.constexpr, +): + # aiter's private helper, by keyword: a renamed or reordered parameter fails to compile + return _qkpv_fp8( + k_nope=kv, + k_rope=kv, + valid=valid, + q_dot=qdot, + m_i=m, + l_i=den, + acc=acc, + head_mask=head_mask, + qk_scale=SCALE * RCP_LN2, + kv_smem=smem, + qk_layout=MF, + pv_layout=MF, + k_layout=gl.DotOperandLayout(1, MF, 8), + v_layout=gl.DotOperandLayout(1, MF, 8), + p_layout=gl.DotOperandLayout(0, MF, 8), + NOPE_DIM=512, + ROPE_DIM=0, + HEAD_SIZE=512, + BLOCK_M=16, + BLOCK_K=BLOCK, + HEAD_ALIGNED=H % 16 == 0, + UNIFORM=True, + HAS_INVALID=True, + ) + + +@gluon.jit +def _compact_tile( + Cache, + Indices, + t, + start, + length, + STRIDE: gl.constexpr, + CAPACITY: gl.constexpr, + PAGE: gl.constexpr, + CACHE_STRIDE: gl.constexpr, + FP4: gl.constexpr, + BUFFER: gl.constexpr, + G: gl.constexpr, + BLOCK: gl.constexpr, +): + col = start + gl.arange(0, BLOCK, layout=gl.SliceLayout(1, G)) + slot = gl.load(Indices + t * STRIDE + col, col < length, -1) + valid = (col < length) & (slot >= 0) & (slot < CAPACITY) + kv = _read_compact_gluon( + Cache, gl.where(valid, slot, 0), valid, PAGE, CACHE_STRIDE, FP4, BUFFER, G + ) + return kv, valid + + +@gluon.jit +def _compact_attention_kernel( + Q, + K, + E, + I, + EI, + L, + EL, + Acc, + Max, + Sum, + Out, + Sink, + Freqs, + Positions, + HAS_SINK: gl.constexpr, + ROPE_DIM: gl.constexpr, + FREQ_STRIDE: gl.constexpr, + QS: gl.constexpr, + QH: gl.constexpr, + H: gl.constexpr, + KS: gl.constexpr, + ES: gl.constexpr, + KP: gl.constexpr, + EP: gl.constexpr, + KN: gl.constexpr, + EN: gl.constexpr, + IS: gl.constexpr, + EIS: gl.constexpr, + NK: gl.constexpr, + NE: gl.constexpr, + KFP4: gl.constexpr, + EFP4: gl.constexpr, + KBUFFER: gl.constexpr, + EBUFFER: gl.constexpr, + SPLITS: gl.constexpr, + SCALE: gl.constexpr, + BLOCK: gl.constexpr, +): + t, split, hg = gl.program_id(0).to(gl.int64), gl.program_id(1), gl.program_id(2) + MF: gl.constexpr = gl.amd.AMDMFMALayout( + version=4, instr_shape=[16, 16, 16], transposed=True, warps_per_cta=[1, 4] + ) + G: gl.constexpr = gl.BlockedLayout([1, 16], [8, 8], [1, 4], [1, 0]) + QG: gl.constexpr = gl.BlockedLayout([1, 8], [8, 8], [1, 4], [1, 0]) + hq = hg * 16 + gl.arange(0, 16, layout=gl.SliceLayout(1, QG)) + dq = gl.arange(0, 512, layout=gl.SliceLayout(0, QG)) + q = gl.load(Q + t * QS + hq[:, None] * QH + dq[None, :], hq[:, None] < H, 0) + qdot = gl.convert_layout(q, gl.DotOperandLayout(0, MF, 8)) + h = hg * 16 + gl.arange(0, 16, layout=gl.SliceLayout(1, MF)) + d = gl.arange(0, 512, layout=gl.SliceLayout(0, MF)) + m = gl.full((16,), -float("inf"), gl.float32, layout=gl.SliceLayout(1, MF)) + den = gl.zeros((16,), gl.float32, layout=gl.SliceLayout(1, MF)) + acc = gl.zeros((16, 512), gl.float32, layout=MF) + smem = gl.allocate_shared_memory( + gl.bfloat16, + [BLOCK, 512], + gl.PaddedSharedLayout.with_identity_for([[512, 8]], [BLOCK, 512], [1, 0]), + ) + for segment in gl.static_range(2): + if segment == 0 or NE > 0: + if segment == 0: + indices, lengths, width, stride, capacity = I, L, NK, IS, KN + cache, page, cache_stride, fp4, buffer = K, KP, KS, KFP4, KBUFFER + else: + indices, lengths, width, stride, capacity = EI, EL, NE, EIS, EN + cache, page, cache_stride, fp4, buffer = E, EP, ES, EFP4, EBUFFER + length = gl.minimum(gl.load(lengths + t), width) + if split * BLOCK < length: + kv, valid = _compact_tile( + cache, + indices, + t, + split * BLOCK, + length, + stride, + capacity, + page, + cache_stride, + fp4, + buffer, + G, + BLOCK, + ) + for start in range((split + SPLITS) * BLOCK, length, SPLITS * BLOCK): + next_kv, next_valid = _compact_tile( + cache, + indices, + t, + start, + length, + stride, + capacity, + page, + cache_stride, + fp4, + buffer, + G, + BLOCK, + ) + m, den, acc = _compact_qkpv( + kv, valid, qdot, m, den, acc, h < H, smem, MF, H, BLOCK, SCALE + ) + kv, valid = next_kv, next_valid + m, den, acc = _compact_qkpv( + kv, valid, qdot, m, den, acc, h < H, smem, MF, H, BLOCK, SCALE + ) + if SPLITS == 1: + if HAS_SINK: + sink = gl.load(Sink + h, h < H, -float("inf")) + scaled_max = m * (SCALE * RCP_LN2) + final_max = gl.maximum(scaled_max, sink * RCP_LN2) + weight = gl.exp2(scaled_max - final_max) + den = den * weight + gl.exp2(sink * RCP_LN2 - final_max) + acc = acc * weight[:, None] + out = (acc / den[:, None]).to(gl.bfloat16) + out = gl.convert_layout(out, QG) + if ROPE_DIM > 0: + pos = gl.load(Positions + t) + is_rope = dq >= 512 - ROPE_DIM + freq_index = ((dq - (512 - ROPE_DIM)) // 2) * 2 + cos = gl.load(Freqs + pos * FREQ_STRIDE + freq_index, is_rope, 0) + sin = gl.load(Freqs + pos * FREQ_STRIDE + freq_index + 1, is_rope, 0) + x = out.to(gl.float32) + even, odd = gl.split(gl.reshape(x, (16, 256, 2))) + swapped = gl.reshape(gl.join(odd, -even), (16, 512)) + rotated = gl.fma(x, cos[None, :], swapped * sin[None, :]) + out = gl.where(is_rope[None, :], rotated.to(gl.bfloat16), out) + gl.store(Out + (t * H + hq[:, None]) * 512 + dq[None, :], out, hq[:, None] < H) + else: + off = (t * SPLITS + split) * H + h + gl.store(Max + off, m * (SCALE * RCP_LN2), h < H) + gl.store(Sum + off, den, h < H) + gl.store(Acc + off[:, None] * 512 + d[None, :], acc, h[:, None] < H) + + +def compact_attention_hip( + q: torch.Tensor, + cache: torch.Tensor, + indices: torch.Tensor, + lengths: torch.Tensor, + sink: Optional[torch.Tensor], + *, + extra_cache: Optional[torch.Tensor] = None, + extra_indices: Optional[torch.Tensor] = None, + extra_lengths: Optional[torch.Tensor] = None, + softmax_scale: float = 512**-0.5, + splits: Optional[int] = None, + inv_rope: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, +) -> torch.Tensor: + """q [n, h, 512] bf16 over the first lengths[t] slots of indices[t] (-1 skipped) in the + packed V41 / V41_FP4 cache [pages, page_size, 1, bytes_per_token], then those of + extra_indices[t] in extra_cache, the sink folded in once. inv_rope is + (freqs_real, positions): the tail of every head is inverse-rotated.""" + assert q.ndim == 3 and q.shape[-1] == 512 and q.dtype == torch.bfloat16 + assert q.stride(-1) == 1 + assert cache.ndim == 4 and cache.shape[2] == 1 + assert cache.shape[-1] in PACKED_TOKEN_BYTES + n, h, _ = q.shape + if n == 0: + return torch.empty_like(q) + indices = indices.reshape(n, -1) + assert indices.stride(-1) == 1 and lengths.is_contiguous() + if extra_cache is None: + assert extra_indices is None and extra_lengths is None + extra_cache, extra_indices, extra_lengths = cache, indices, lengths + ne = 0 + else: + assert extra_cache.shape[-1] in PACKED_TOKEN_BYTES + extra_indices = extra_indices.reshape(n, -1) + assert extra_indices.stride(-1) == 1 and extra_lengths.is_contiguous() + ne = extra_indices.shape[1] + if splits is None: + # enough splits to fill the CUs, none past the slots a row has + num_cus = torch.cuda.get_device_properties(q.device).multi_processor_count + splits = min( + MAX_SPLITS, + triton.cdiv(indices.shape[1] + ne, TILE_SLOTS), + triton.cdiv(num_cus, n * triton.cdiv(h, 16)), + ) + assert splits > 0 + out = torch.empty((n, h, 512), dtype=q.dtype, device=q.device) if splits == 1 else q + if inv_rope is not None: + freqs, positions = inv_rope + assert freqs.dtype == torch.float32 and freqs.stride(1) == 1 + assert positions.shape == (n,) + rope_dim = freqs.shape[1] + assert 0 < rope_dim <= 512 and rope_dim % 2 == 0 + else: + freqs = positions = out + rope_dim = 0 + if splits == 1: + acc = maximum = denominator = out + else: + acc = torch.empty((n, splits, h, 512), dtype=torch.float32, device=q.device) + maximum = torch.empty((n, splits, h), dtype=torch.float32, device=q.device) + denominator = torch.empty_like(maximum) + _compact_attention_kernel[(n, splits, triton.cdiv(h, 16))]( + q, + cache.view(torch.uint8), + extra_cache.view(torch.uint8), + indices, + extra_indices, + lengths, + extra_lengths, + acc, + maximum, + denominator, + out, + sink if sink is not None else out, + freqs, + positions, + HAS_SINK=sink is not None, + ROPE_DIM=rope_dim, + FREQ_STRIDE=freqs.stride(0), + QS=q.stride(0), + QH=q.stride(1), + H=h, + KS=cache.stride(0), + ES=extra_cache.stride(0), + KP=cache.shape[1], + EP=extra_cache.shape[1], + KN=cache.shape[0] * cache.shape[1], + EN=extra_cache.shape[0] * extra_cache.shape[1], + IS=indices.stride(0), + EIS=extra_indices.stride(0), + NK=indices.shape[1], + NE=ne, + KFP4=cache.shape[-1] == KVLayout.V41_FP4.bytes_per_token, + EFP4=extra_cache.shape[-1] == KVLayout.V41_FP4.bytes_per_token, + KBUFFER=cache.shape[0] * cache.stride(0) < 2**31, + EBUFFER=extra_cache.shape[0] * extra_cache.stride(0) < 2**31, + SPLITS=splits, + SCALE=softmax_scale, + BLOCK=TILE_SLOTS, + num_warps=4, + ) + if splits == 1: + return out + return gfx_sparse_split_reduce( + acc, maximum, denominator, sink, q.dtype, inv_rope=inv_rope + ) diff --git a/python/sglang/kernels/ops/attention/dsv4/decode_attention_sm100.py b/python/sglang/kernels/ops/attention/dsv4/decode_attention_sm100.py index a64d1e23be..f0e57b9496 100644 --- a/python/sglang/kernels/ops/attention/dsv4/decode_attention_sm100.py +++ b/python/sglang/kernels/ops/attention/dsv4/decode_attention_sm100.py @@ -1,5 +1,5 @@ """SM100 small-batch paged attention with the heads on the MMA N dimension; the -caller applies the inverse RoPE to the result.""" +inverse RoPE is applied by the caller, or by the combine when inv_rope is given.""" from typing import Optional @@ -37,6 +37,23 @@ def can_use_swapab_attention( ) +@triton.jit +def inverse_rope_tail( + out, d, FREQS, pos, freq_stride, HEAD_DIM: tl.constexpr, ROPE_DIM: tl.constexpr +): + """out (head dims d) with its last ROPE_DIM dims inverse-rotated at position pos. It + rotates the rounded values, as the model's standalone RoPE kernel reads them.""" + x = out.to(tl.float32) + is_rope = d >= HEAD_DIM - ROPE_DIM + freq_index = ((d - (HEAD_DIM - ROPE_DIM)) // 2) * 2 + cos = tl.load(FREQS + pos * freq_stride + freq_index, is_rope, 0) + sin = tl.load(FREQS + pos * freq_stride + freq_index + 1, is_rope, 0) + signed = tl.where(d % 2 == 0, -x * sin, x * sin) + swapped = tl.reshape(tl.flip(tl.reshape(signed, (d.shape[0] // 2, 2)), 1), d.shape) + rotated = tl.fma(x, cos, swapped) + return tl.where(is_rope, rotated.to(out.dtype), out) + + @triton.jit def _combine( PART, @@ -44,6 +61,10 @@ def _combine( SUM, SINK, OUT, + FREQS, + POSITIONS, + FREQ_STRIDE: tl.constexpr, + ROPE_DIM: tl.constexpr, NT: tl.constexpr, ST: tl.constexpr, H: tl.constexpr, @@ -66,6 +87,10 @@ def _combine( ) out = tl.sum(vals * factor[:, None], 0) / denominator out = tl.where((denominator > 0) & (sink != float("inf")), out, 0.0) + out = out.to(OUT.dtype.element_ty) + if ROPE_DIM > 0: + pos = tl.load(POSITIONS + b) + out = inverse_rope_tail(out, d, FREQS, pos, FREQ_STRIDE, 512, ROPE_DIM) tl.store(OUT + (b * H + h) * 512 + d, out) @@ -78,10 +103,14 @@ def swapab_attention( extra_kv=None, extra_indices=None, extra_lengths=None, + inv_rope=None, + partial_gluon=None, ): """V4-layout attention on 16 heads; `extra_*` is a second slot range appended - to each request's keys, and the attention sink is folded in exactly once.""" - from .decode_attention_sm100_gluon import partial_gluon + to each request's keys, and the attention sink is folded in exactly once. + `partial_gluon` replaces the SM100 split-KV kernel (swapab_gluon_hip passes gfx950's).""" + if partial_gluon is None: + from .decode_attention_sm100_gluon import partial_gluon block = 64 b, h, d = q.shape[0], q.shape[-2], q.shape[-1] @@ -144,6 +173,15 @@ def swapab_attention( TILE=LAYOUT.tile_size, num_warps=4, ) + if inv_rope is not None: + freqs, positions = inv_rope + assert freqs.dtype == torch.float32 and freqs.stride(1) == 1 + assert positions.shape == (b,) + rope_dim = freqs.shape[1] + assert 0 < rope_dim <= 512 and rope_dim % 2 == 0 + else: + freqs = positions = out + rope_dim = 0 bd = 64 if ne else 512 _combine[(b, h, triton.cdiv(512, bd))]( partial, @@ -151,6 +189,10 @@ def swapab_attention( sums, sink, out, + freqs, + positions, + FREQ_STRIDE=freqs.stride(0), + ROPE_DIM=rope_dim, NT=nt, ST=triton.next_power_of_2(nt), H=h, diff --git a/python/sglang/kernels/ops/attention/dsv4/q_rope_store.py b/python/sglang/kernels/ops/attention/dsv4/q_rope_store.py index b4ac682d4b..e2073b9e59 100644 --- a/python/sglang/kernels/ops/attention/dsv4/q_rope_store.py +++ b/python/sglang/kernels/ops/attention/dsv4/q_rope_store.py @@ -5,11 +5,19 @@ import triton import triton.language as tl from sglang.kernels.jit.utils import is_arch_support_pdl +from sglang.srt.utils import is_hip @triton.jit def _q_rope_store( - X, Y, F, POS, SX: tl.constexpr, SY: tl.constexpr, USE_GDC: tl.constexpr = False + X, + Y, + F, + POS, + SX: tl.constexpr, + SY: tl.constexpr, + IS_HIP: tl.constexpr, + USE_GDC: tl.constexpr = False, ): row, head = tl.program_id(0), tl.program_id(1) r = tl.arange(0, 512) @@ -22,7 +30,11 @@ def _q_rope_store( sin = tl.load(F + position * 64 + (r - 448) // 2 * 2 + 1, r >= 448, 0) # Match the operation order of deepseek_rope_kernel before BF16 rounding. even = tl.fma(value, cos, -partner * sin) - odd = tl.fma(partner, sin, value * cos) + if IS_HIP: + # Match AMD flat RoPE: round the sine product before the cosine FMA. + odd = tl.fma(value, cos, partner * sin) + else: + odd = tl.fma(partner, sin, value * cos) rotated = tl.where((r & 1) == 0, even, odd) tl.store(Y + row * SY + head * 512 + r, tl.where(r >= 448, rotated, value)) if USE_GDC: @@ -40,6 +52,7 @@ def _q_rope_store_prefill( SX: tl.constexpr, SY: tl.constexpr, BLOCK_HEADS: tl.constexpr, + IS_HIP: tl.constexpr, ): # Keep token count dynamic to avoid compiling every prefill batch length. heads = tl.program_id(0) * BLOCK_HEADS + tl.arange(0, BLOCK_HEADS) @@ -56,7 +69,11 @@ def _q_rope_store_prefill( sin = tl.load(F + freq_offset + 1, rope_mask, 0) # Keep the same arithmetic and BF16 rounding as the decode kernel. even = tl.fma(value, cos, -partner * sin) - odd = tl.fma(partner, sin, value * cos) + if IS_HIP: + # Match AMD flat RoPE: round the sine product before the cosine FMA. + odd = tl.fma(value, cos, partner * sin) + else: + odd = tl.fma(partner, sin, value * cos) rotated = tl.where((r[None, :] & 1) == 0, even, odd) y_offset = row[:, None].to(tl.int64) * SY + head[:, None] * 512 + r[None, :] tl.store( @@ -91,6 +108,7 @@ def q_rope_store( q.stride(0), output.stride(0), BLOCK_HEADS=4, + IS_HIP=is_hip(), num_warps=4, ) return @@ -102,6 +120,7 @@ def q_rope_store( positions, q.stride(0), output.stride(0), + IS_HIP=is_hip(), num_warps=4, **pdl_kwargs, ) diff --git a/python/sglang/kernels/ops/attention/dsv4/swapab_gluon_hip.py b/python/sglang/kernels/ops/attention/dsv4/swapab_gluon_hip.py new file mode 100644 index 0000000000..2ba22a94b6 --- /dev/null +++ b/python/sglang/kernels/ops/attention/dsv4/swapab_gluon_hip.py @@ -0,0 +1,130 @@ +"""Native 16-head MFMA layouts for gfx950 paged attention.""" + +import functools + +from triton.experimental import gluon +from triton.experimental.gluon import language as gl + +from .decode_attention_sm100 import swapab_attention as _swapab_attention + +# the V4 page load is plain gluon (no Blackwell ops), so the SM100 kernel's serves gfx950 too +from .decode_attention_sm100_gluon import _load_v4 + + +@gluon.jit +def partial_gluon( + Q, + K, + E, + IDX, + EI, + L, + EL, + PART, + MAX, + SUM, + QS: gl.constexpr, + QH: gl.constexpr, + IS: gl.constexpr, + EIS: gl.constexpr, + KP: gl.constexpr, + KS: gl.constexpr, + EP: gl.constexpr, + ES: gl.constexpr, + NK: gl.constexpr, + NE: gl.constexpr, + NT: gl.constexpr, + KT: gl.constexpr, + BT: gl.constexpr, + H: gl.constexpr, + SCALE: gl.constexpr, + KTOKENS: gl.constexpr, + ETOKENS: gl.constexpr, + COMPENSATE: gl.constexpr, + SWAP_AB: gl.constexpr, + DATA_BYTES: gl.constexpr, + SCALE_BYTES: gl.constexpr, + TILE: gl.constexpr, +): + gl.static_assert(SWAP_AB and H == 16 and (BT == 64 or BT == 128)) + b, t = gl.program_id(0), gl.program_id(1) + kv_layout: gl.constexpr = gl.BlockedLayout([1, 8], [8, 8], [1, 4], [1, 0]) + n = gl.arange(0, BT, gl.SliceLayout(1, kv_layout)) + if t < KT: + at = t * BT + n + length = gl.load(L + b) + ids = gl.load(IDX + b * IS + at, at < NK, -1) + valid = (at < NK) & (at < length) & (ids >= 0) & (ids < KTOKENS) + kv = _load_v4( + K, + gl.maximum(ids, 0), + valid, + KP, + KS, + kv_layout, + DATA_BYTES, + SCALE_BYTES, + TILE, + ) + else: + at = (t - KT) * BT + n + length = gl.load(EL + b) + ids = gl.load(EI + b * EIS + at, at < NE, -1) + valid = (at < NE) & (at < length) & (ids >= 0) & (ids < ETOKENS) + kv = _load_v4( + E, + gl.maximum(ids, 0), + valid, + EP, + ES, + kv_layout, + DATA_BYTES, + SCALE_BYTES, + TILE, + ) + qh = gl.arange(0, H, gl.SliceLayout(1, kv_layout)) + qd = gl.arange(0, 512, gl.SliceLayout(0, kv_layout)) + q = gl.load(Q + b * QS + qh[:, None] * QH + qd[None, :]) + mf: gl.constexpr = gl.amd.AMDMFMALayout( + version=4, instr_shape=[16, 16, 16], transposed=True, warps_per_cta=[4, 1] + ) + scores = ( + gl.amd.cdna4.mfma( + gl.convert_layout(kv, gl.DotOperandLayout(0, mf, 8)), + gl.convert_layout(q.permute((1, 0)), gl.DotOperandLayout(1, mf, 8)), + gl.zeros((BT, H), gl.float32, mf), + ) + * SCALE + ) + valid = gl.convert_layout(valid, gl.SliceLayout(1, mf)) + scores = gl.where(valid[:, None], scores, -float("inf")) + mx = gl.max(scores, 0) + mx = gl.where(mx == -float("inf"), 0.0, mx) + prob = gl.exp(scores - mx[None, :]) + denom = gl.sum(prob, 0) + hi = prob.to(gl.bfloat16) + value = gl.amd.cdna4.mfma( + gl.convert_layout(kv.permute((1, 0)), gl.DotOperandLayout(0, mf, 8)), + gl.convert_layout(hi, gl.DotOperandLayout(1, mf, 8)), + gl.zeros((512, H), gl.float32, mf), + ) + if COMPENSATE: + lo = (prob - hi.to(gl.float32)).to(gl.bfloat16) + value = gl.amd.cdna4.mfma( + gl.convert_layout(kv.permute((1, 0)), gl.DotOperandLayout(0, mf, 8)), + gl.convert_layout(lo, gl.DotOperandLayout(1, mf, 8)), + value, + ) + hd: gl.constexpr = gl.BlockedLayout([1, 8], [8, 8], [1, 4], [1, 0]) + value = gl.convert_layout(value.permute((1, 0)), hd) + h = gl.arange(0, H, gl.SliceLayout(1, hd)) + d = gl.arange(0, 512, gl.SliceLayout(0, hd)) + gl.store(PART + ((b * NT + t) * H + h[:, None]) * 512 + d[None, :], value) + stat: gl.constexpr = gl.BlockedLayout([1], [64], [4], [0]) + hs = gl.arange(0, H, stat) + gl.store(MAX + (b * NT + t) * H + hs, gl.convert_layout(mx, stat)) + gl.store(SUM + (b * NT + t) * H + hs, gl.convert_layout(denom, stat)) + + +# the SM100 host path (sink, combine, inverse RoPE) on the gfx950 split-KV kernel above +swapab_attention = functools.partial(_swapab_attention, partial_gluon=partial_gluon) diff --git a/python/sglang/kernels/ops/attention/sparse_decode_reduce_hip.py b/python/sglang/kernels/ops/attention/sparse_decode_reduce_hip.py new file mode 100644 index 0000000000..4b6a570dfd --- /dev/null +++ b/python/sglang/kernels/ops/attention/sparse_decode_reduce_hip.py @@ -0,0 +1,133 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Split-KV combine for aiter's gfx950 sparse decode kernel, bitwise _pa_decode_sparse_reduce, +with the DeepSeek-V4 inverse RoPE folded in.""" + +from __future__ import annotations + +from typing import Optional, Tuple + +import torch +import triton +import triton.language as tl + +from sglang.kernels.ops.attention.dsv4.decode_attention_sm100 import inverse_rope_tail + + +@triton.jit +def _gfx_sparse_decode_reduce_kernel( + part_m_ptr, + part_l_ptr, + part_acc_ptr, + sink_ptr, + out_ptr, + fr_ptr, + pos_ptr, + pm_stride_t, + pm_stride_s, + pa_stride_t, + pa_stride_s, + pa_stride_h, + out_stride_t, + out_stride_h, + fr_stride_pos, + NUM_SPLITS: tl.constexpr, + D: tl.constexpr, + RD: tl.constexpr, + HAS_SINK: tl.constexpr, + INV_ROPE: tl.constexpr, +): + """Grid (num_queries, num_heads). Partials: part_m / part_l + [T, S, H] fp32 (m in the base-2 exponent domain), part_acc + [T, S, H, D] fp32 -- the layout aiter's gluon main kernel stores.""" + RCP_LN2: tl.constexpr = 1.4426950408889634 + t = tl.program_id(0) + h = tl.program_id(1) + d = tl.arange(0, D) + pm_base = t * pm_stride_t + h + pa_base = t * pa_stride_t + h * pa_stride_h + + neg_inf = float("-inf") + m_final = tl.full((), neg_inf, tl.float32) + for s in tl.static_range(NUM_SPLITS): + m_s = tl.load(part_m_ptr + pm_base + s * pm_stride_s) + m_final = tl.maximum(m_final, m_s) + if HAS_SINK: + sink = tl.load(sink_ptr + h).to(tl.float32) + m_final = tl.maximum(m_final, sink * RCP_LN2) + + l_final = tl.full((), 0.0, tl.float32) + acc = tl.zeros([D], dtype=tl.float32) + for s in tl.static_range(NUM_SPLITS): + m_s = tl.load(part_m_ptr + pm_base + s * pm_stride_s) + l_s = tl.load(part_l_ptr + pm_base + s * pm_stride_s) + w = tl.exp2(m_s - m_final) + # gluon contracts l + w * l_s into one fma; Triton leaves the scalar chain as mul + add + l_final = tl.fma(w, l_s, l_final) + acc_s = tl.load(part_acc_ptr + pa_base + s * pa_stride_s + d) + acc = tl.fma(w, acc_s, acc) + if HAS_SINK: + l_final = l_final + tl.exp2(sink * RCP_LN2 - m_final) + + out = (acc / l_final).to(out_ptr.dtype.element_ty) + if INV_ROPE: + pos = tl.load(pos_ptr + t) + out = inverse_rope_tail(out, d, fr_ptr, pos, fr_stride_pos, D, RD) + tl.store(out_ptr + t * out_stride_t + h * out_stride_h + d, out) + + +def gfx_sparse_split_reduce( + part_acc: torch.Tensor, + part_m: torch.Tensor, + part_l: torch.Tensor, + attn_sink: Optional[torch.Tensor], + out_dtype: torch.dtype = torch.bfloat16, + inv_rope: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, +) -> torch.Tensor: + """Combine pa_decode_sparse(..., skip_reduce=True)'s partials into the + [T, H, D] output. inv_rope is (freqs_real [max_pos, rope_dim] fp32, + positions [T]): the last rope_dim of every head are inverse-rotated.""" + T, S, H, D = part_acc.shape + assert part_m.shape == (T, S, H) and part_l.shape == (T, S, H), ( + part_m.shape, + part_l.shape, + ) + assert part_acc.stride(3) == 1 and part_m.stride(2) == 1 and part_l.stride(2) == 1 + assert part_m.stride() == part_l.stride() + assert triton.next_power_of_2(D) == D and S >= 1 + out = torch.empty((T, H, D), dtype=out_dtype, device=part_acc.device) + if T == 0: + return out + has_sink = attn_sink is not None + if inv_rope is not None: + freqs_real, positions = inv_rope + rope_dim = freqs_real.shape[1] + assert freqs_real.dtype == torch.float32 and freqs_real.stride(1) == 1 + assert positions.shape == (T,), positions.shape + assert rope_dim % 2 == 0 and rope_dim <= D + else: + freqs_real = positions = out # unread placeholders + rope_dim = 0 + _gfx_sparse_decode_reduce_kernel[(T, H)]( + part_m, + part_l, + part_acc, + attn_sink if has_sink else out, + out, + freqs_real, + positions, + part_m.stride(0), + part_m.stride(1), + part_acc.stride(0), + part_acc.stride(1), + part_acc.stride(2), + out.stride(0), + out.stride(1), + freqs_real.stride(0), + NUM_SPLITS=S, + D=D, + RD=rope_dim, + HAS_SINK=has_sink, + INV_ROPE=inv_rope is not None, + num_warps=1, + ) + return out diff --git a/test/registered/kernels/ops/attention/dsv4/test_compact_attention_hip.py b/test/registered/kernels/ops/attention/dsv4/test_compact_attention_hip.py new file mode 100644 index 0000000000..50122bab45 --- /dev/null +++ b/test/registered/kernels/ops/attention/dsv4/test_compact_attention_hip.py @@ -0,0 +1,103 @@ +"""`compact_attention_hip` against a torch softmax-with-sink over the dequantized pages.""" + +import itertools +import unittest + +import torch + +from sglang.kernels.ops.attention.dsv4.compact_attention_hip import ( + compact_attention_hip, +) +from sglang.srt.utils import is_gfx95_supported, is_hip +from sglang.test.ci.ci_register import register_amd_ci +from sglang.test.test_utils import CustomTestCase + +register_amd_ci(est_time=20, stage="stage-b", runner_config="1-gpu-small-amd-mi35x") + +PAGE, PAGES = 64, 8 +E2M1 = [0, 0.5, 1, 1.5, 2, 3, 4, 6, -0.0, -0.5, -1, -1.5, -2, -3, -4, -6] + + +def _cache(fp4: bool) -> torch.Tensor: + """[pages, page, 1, bytes]: 512 fp8 + 16 ue8m0 per token (528 B), or 512 e2m1 + 32 e4m3 (288 B).""" + n = PAGES * PAGE + if fp4: + data = torch.randint(0, 256, (PAGES, PAGE * 256), dtype=torch.uint8) + scale = (torch.rand(PAGES, PAGE * 32) * 0.5 + 0.25).to(torch.float8_e4m3fn) + else: + data = torch.randn(PAGES, PAGE * 512).to(torch.float8_e4m3fn).view(torch.uint8) + scale = torch.randint(120, 130, (PAGES, PAGE * 16), dtype=torch.uint8) + flat = torch.cat([data, scale.view(torch.uint8)], dim=1) + return flat.view(PAGES, PAGE, 1, flat.shape[1] // PAGE).cuda().contiguous() + + +def _dequant(cache: torch.Tensor) -> torch.Tensor: + """[pages * page, 512] fp32 rows, rounded to bf16 as the kernel reads them.""" + flat = cache.view(PAGES, -1).cpu() + if cache.shape[-1] == 288: + packed = flat[:, : PAGE * 256].view(PAGES, PAGE, 256).long() + lut = torch.tensor(E2M1) + v = torch.stack([lut[packed & 15], lut[packed >> 4]], -1).view(PAGES, PAGE, 512) + s = flat[:, PAGE * 256 :].view(PAGES, PAGE, 32).view(torch.float8_e4m3fn) + v = v * s.float().repeat_interleave(16, -1) + else: + v = ( + flat[:, : PAGE * 512] + .view(PAGES, PAGE, 512) + .view(torch.float8_e4m3fn) + .float() + ) + e = flat[:, PAGE * 512 :].view(PAGES, PAGE, 16).float() + v = v * torch.exp2(e - 127).repeat_interleave(32, -1) + return v.to(torch.bfloat16).float().view(-1, 512).cuda() + + +@unittest.skipUnless(is_hip() and is_gfx95_supported(), "gfx950 gluon MFMA kernel") +class TestCompactAttentionHip(CustomTestCase): + def test_matches_torch_over_both_caches_splits_and_empty_rows(self): + """Both packed layouts, a second cache, -1 slots and a row with no keys (sink only).""" + torch.manual_seed(0) + n, h, nk, ne, scale = 5, 16, 256, 128, 512**-0.5 + for fp4, splits in itertools.product((False, True), (1, 4)): + main, extra = _cache(fp4), _cache(not fp4) + q = torch.randn(n, h, 512, device="cuda").bfloat16() + idx = torch.randint( + 0, PAGES * PAGE, (n, nk), device="cuda", dtype=torch.int32 + ) + idx[:, ::7] = -1 + lens = torch.randint(1, nk + 1, (n,), device="cuda", dtype=torch.int32) + eidx = torch.randint( + 0, PAGES * PAGE, (n, ne), device="cuda", dtype=torch.int32 + ) + elens = torch.randint(1, ne + 1, (n,), device="cuda", dtype=torch.int32) + lens[0] = elens[0] = 0 + sink = torch.randn(h, device="cuda") + + out = compact_attention_hip( + q, + main, + idx, + lens, + sink, + extra_cache=extra, + extra_indices=eidx, + extra_lengths=elens, + softmax_scale=scale, + splits=splits, + ).float() + + kd, ed = _dequant(main), _dequant(extra) + for t in range(n): + rows = [idx[t, : lens[t]], eidx[t, : elens[t]]] + kv = torch.cat( + [kd[rows[0][rows[0] >= 0].long()], ed[rows[1][rows[1] >= 0].long()]] + ) + s = torch.cat([q[t].float() @ kv.T * scale, sink[:, None]], 1) + ref = torch.softmax(s, 1)[:, :-1] @ kv + torch.testing.assert_close( + out[t], ref, rtol=2e-2, atol=1e-2, msg=f"{fp4=} {splits=} row {t}" + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/kernels/ops/attention/dsv4/test_dsv41_decode_glue_hip.py b/test/registered/kernels/ops/attention/dsv4/test_dsv41_decode_glue_hip.py new file mode 100644 index 0000000000..57c7d1b050 --- /dev/null +++ b/test/registered/kernels/ops/attention/dsv4/test_dsv41_decode_glue_hip.py @@ -0,0 +1,358 @@ +"""The V4.1 HIP decode glue is bitwise the torch chains it replaces and selects within each row's reach.""" + +from __future__ import annotations + +import random +import sys + +import pytest +import torch + +from sglang.kernels.ops.attention.dsv4.attn_glue_hip import ( + expand_index_page_table, + low_ratio_compression_metadata, +) +from sglang.kernels.ops.attention.dsv4.candidate_blocks import ( + select_candidate_block_ids, +) +from sglang.kernels.ops.attention.dsv4.candidate_blocks_hip import ( + CandidateBlocks, + select_candidate_blocks_hip, + topk_within_candidate_blocks_hip, +) +from sglang.srt.layers.attention.deepseek_v4_backend import ( + _low_ratio_compression_metadata, +) +from sglang.srt.layers.attention.dsv4.metadata import ( + expand_index_page_table as _expand_index_page_table, +) +from sglang.srt.utils import is_gfx95_supported, is_hip +from sglang.test.ci.ci_register import register_amd_ci +from sglang.test.test_utils import CustomTestCase + +register_amd_ci(est_time=15, stage="stage-b", runner_config="1-gpu-small-amd-mi35x") + +pytestmark = pytest.mark.skipif( + not (is_hip() and is_gfx95_supported()), + reason="HIP decode glue for the AITER / FlyDSL DeepSeek-V4.1 path (gfx95x).", +) + +DEVICE = "cuda" + +INDEX_PAGE_SIZE = 64 +TOPK = 512 +# Released V4.1 config; the span 2048 x 8 = 16384 is where level one starts to bind. +TOPK_BLOCKS, BLOCK_SIZE = 2048, 8 + + +def index_slots(page_table, pos): + """Slot of compressed position pos through the expanded indexer page table.""" + return ( + page_table.gather(1, pos // INDEX_PAGE_SIZE) * INDEX_PAGE_SIZE + + pos % INDEX_PAGE_SIZE + ) + + +def reference_position_mask(logits, lens, topk_blocks, block_size): + """The reference's level one on logits whose tail past the reach is -inf.""" + col = torch.arange(logits.shape[1], device=logits.device) + pre = logits.masked_fill(col >= lens[:, None], -torch.inf) + ids = select_candidate_block_ids(pre, lens[:, None], topk_blocks, block_size) + return ids_to_position_mask(ids, block_size, logits.shape[1]) + + +def candidate_block_ids_to_mask(ids, num_blocks): + """bool [rows, num_blocks] block mask of CandidateBlocks.ids.""" + rows = ids.shape[0] + # column num_blocks is the sink for the -1 padding; the mask is the view before it + keep = torch.zeros((rows, num_blocks + 1), dtype=torch.bool, device=ids.device) + keep.scatter_(1, ids.masked_fill(ids < 0, num_blocks).to(torch.int64), True) + return keep[:, :num_blocks] + + +def ids_to_position_mask(ids, block_size, width): + num_blocks = (width + block_size - 1) // block_size + keep = candidate_block_ids_to_mask(ids, num_blocks) + return keep.repeat_interleave(block_size, dim=-1)[:, :width] + + +def reference_consumer_rows(logits, lens, pos_mask, topk): + """Per row: the set the reference consumer selects among the reachable candidates + and its valid count; tie-free logits make it exact.""" + col = torch.arange(logits.shape[1], device=logits.device) + out = [] + for b in range(logits.shape[0]): + n = int(lens[b]) + cand = pos_mask[b] & (col < n) + n_cand = int(cand.sum()) + k = min(topk, n_cand) + s = logits[b].masked_fill(~cand, -torch.inf) + out.append((set(s.topk(k).indices.tolist()), k)) + return out + + +class TestTwoLevelDecodeHip(CustomTestCase): + """Level one of the two-level top-k: the source keeps TOPK_BLOCKS x BLOCK_SIZE positions, later ratio-1 sources select inside them.""" + + def _garbage_tail(self, logits, lens): + """Kernel garbage past each row's reach (large positives on even rows, NaN on + odd), so a helper reading the tail fails loudly.""" + col = torch.arange(logits.shape[1], device=logits.device) + tail = col[None, :] >= lens[:, None] + odd = (torch.arange(logits.shape[0], device=logits.device) % 2 == 1)[:, None] + logits = logits.masked_fill(tail & ~odd, 1e4) + return logits.masked_fill(tail & odd, torch.nan) + + def _assert_consumer_matches_reference(self, logits, seq, cands, page_table, msg): + rows, width = logits.shape + page_indices = torch.full((rows, TOPK), 7, dtype=torch.int32, device="cuda") + raw_indices = torch.full((rows, TOPK), 7, dtype=torch.int32, device="cuda") + topk_within_candidate_blocks_hip( + logits, + seq, + cands, + page_table=page_table, + page_size=INDEX_PAGE_SIZE, + page_indices=page_indices, + raw_indices=raw_indices, + ) + pos_mask = ids_to_position_mask(cands.ids, cands.block_size, width) + for b, (want, k) in enumerate( + reference_consumer_rows(logits, seq, pos_mask, TOPK) + ): + got = raw_indices[b] + self.assertTrue(bool((got[:k] >= 0).all()), f"{msg}: prefix row {b}") + self.assertTrue(bool((got[k:] == -1).all()), f"{msg}: padding row {b}") + self.assertEqual(set(got[:k].tolist()), want, f"{msg}: selection row {b}") + sel = got[:k].to(torch.int64) + expect = index_slots(page_table[b : b + 1], sel[None])[0] + self.assertTrue( + torch.equal(page_indices[b, :k].to(torch.int64), expect), + f"{msg}: slots row {b}", + ) + self.assertTrue(bool((page_indices[b, k:] == -1).all())) + + def test_level_one_matches_reference_under_garbage_tail(self): + """The HIP block top-k (AOT row-split and torch fallback) must publish the + reference's blocks and never read past a row's reach.""" + torch.manual_seed(11) + cases = ( + # Released blocks, a rectangle just wider than the longest row. + (TOPK_BLOCKS, BLOCK_SIZE, 40000, [3, 8, 16384, 16385, 20000, 40000, 1]), + # Released blocks on a 1M-wide rectangle, the page table's capacity on + # a 1M-context server: the block top-k takes the AOT row-split path. + (TOPK_BLOCKS, BLOCK_SIZE, 1 << 20, [16385, 131072, 7, 600]), + ) + for topk_blocks, block_size, width, lens in cases: + with self.subTest(topk_blocks=topk_blocks, width=width, lens=lens): + seq = torch.tensor(lens, dtype=torch.int32, device="cuda") + raw = torch.randn(len(lens), width, device="cuda") + raw = self._garbage_tail(raw, seq) + expected = reference_position_mask(raw, seq, topk_blocks, block_size) + + cands = select_candidate_blocks_hip( + raw, seq, topk_blocks=topk_blocks, block_size=block_size + ) + ids = cands.ids + self.assertEqual(ids.shape, (len(lens), topk_blocks)) + self.assertTrue( + torch.equal( + cands.compact_lens.cpu(), + torch.tensor( + [ + min((n + block_size - 1) // block_size, topk_blocks) + * block_size + for n in lens + ], + dtype=torch.int32, + ), + ) + ) + self.assertEqual(ids.dtype, torch.int32) + got = ids_to_position_mask(ids, block_size, width) + self.assertTrue(torch.equal(got, expected), "published blocks") + for b, n in enumerate(lens): + row = ids[b] + n_ids = int((row >= 0).sum()) + self.assertTrue(bool((row[:n_ids] >= 0).all()), "padding last") + self.assertLessEqual( + int(got[b, :n].sum()), topk_blocks * block_size + ) + + n_pages = (width + INDEX_PAGE_SIZE - 1) // INDEX_PAGE_SIZE + page_table = torch.stack( + [torch.randperm(n_pages, device="cuda") for _ in lens] + ).to(torch.int32) + self._assert_consumer_matches_reference( + raw, seq, cands, page_table, "consumer" + ) + + +def _seed(seed: int) -> random.Random: + torch.manual_seed(seed) + return random.Random(seed) + + +def _topk_inputs(bs, width, page_size, lens): + # a row longer than its logits reads past the tensor (scores[0, len) and + # page_table[0, len // page_size]), and what it selects there differs by launch + assert max(lens, default=0) <= width, (lens, width) + # distinct scores per row: the radix top-k breaks a tie at the threshold in + # atomic-counter order, so two launches over ties can select different sets + scores = ( + torch.stack([torch.randperm(width, device=DEVICE).float() for _ in range(bs)]) + * 0.37 + ) + seq_lens = torch.tensor(lens, dtype=torch.int32, device=DEVICE) + n_pages = (width + page_size - 1) // page_size + page_table = torch.randint( + 0, 1 << 20, (bs, n_pages), dtype=torch.int32, device=DEVICE + ) + return scores, seq_lens, page_table + + +def _sort_rows(page, raw): + """Rows ascending by position (by slot without raw), -1 padding last.""" + key = (raw if raw is not None else page).to(torch.int64) + order = key.masked_fill(key < 0, torch.iinfo(torch.int64).max).argsort(dim=1) + return page.gather(1, order), (raw.gather(1, order) if raw is not None else None) + + +def _candidates(rng, rows, num_blocks, topk_blocks, block_size, seq_lens): + ids = torch.full((rows, topk_blocks), -1, dtype=torch.int32, device=DEVICE) + for r in range(rows): + reach = min(num_blocks, -(-int(seq_lens[r]) // block_size)) + picks = rng.sample(range(reach), min(reach, topk_blocks)) + rng.shuffle(picks) + if picks: + ids[r, : len(picks)] = torch.tensor(picks, dtype=torch.int32, device=DEVICE) + block_lens = (seq_lens + block_size - 1) // block_size + compact_lens = (torch.clamp(block_lens, max=topk_blocks) * block_size).to( + torch.int32 + ) + width = topk_blocks * block_size + return CandidateBlocks( + ids=ids, + compact_lens=compact_lens, + compact_page_table=torch.zeros((rows, 1), dtype=torch.int32, device=DEVICE), + compact_page_size=1 << (width - 1).bit_length(), + block_size=block_size, + ) + + +@pytest.mark.parametrize("topk", [512], ids=["512"]) +@pytest.mark.parametrize("with_raw", [True, False], ids=["True", "False"]) +def test_sorted_candidate_mapping_matches_pack_then_sort(topk: int, with_raw: bool): + rng = _seed(11 + topk) + block_size, topk_blocks, page_size = 64, 16, 64 + for rows in (1, 7): + width = 8192 + lens = [ + rng.choice([0, 1, 300, topk, topk + 5, 3000, width]) for _ in range(rows) + ] + scores, seq_lens, page_table = _topk_inputs(rows, width, page_size, lens) + cands = _candidates( + rng, rows, width // block_size, topk_blocks, block_size, seq_lens + ) + outs = [] + for sort in (False, True): + page = torch.empty(rows, topk, dtype=torch.int32, device=DEVICE) + raw = torch.empty_like(page) if with_raw else None + topk_within_candidate_blocks_hip( + scores, + seq_lens, + cands, + page_table=page_table, + page_size=page_size, + page_indices=page, + raw_indices=raw, + sort_output=sort, + ) + if not sort: + page, raw = _sort_rows(page, raw) + outs.append((page, raw)) + assert torch.equal(outs[0][0], outs[1][0]) + if with_raw: + assert torch.equal(outs[0][1], outs[1][1]) + + +@pytest.mark.parametrize("bpp", [4], ids=["4"]) +def test_expand_index_page_table(bpp: int): + _seed(5) + for bs, n in ((1, 4608), (3, 17), (0, 10)): + page_table = torch.randint( + 0, 1 << 20, (bs, n), dtype=torch.int32, device=DEVICE + ) + ref = _expand_index_page_table( + page_table, full_page_size=64 * bpp, compress_ratio=1, index_page_size=64 + ) + out = expand_index_page_table(page_table, bpp) + assert out.dtype is torch.int32 and out.shape == ref.shape + assert torch.equal(out, ref) + # a strided (row-sliced) table is read through its strides + page_table = torch.randint(0, 1 << 20, (8, 33), dtype=torch.int32, device=DEVICE)[ + ::2 + ] + assert torch.equal( + expand_index_page_table(page_table, 4), + _expand_index_page_table( + page_table, full_page_size=256, compress_ratio=1, index_page_size=64 + ), + ) + + +@pytest.mark.parametrize("loc_dtype", [torch.int64], ids=["torch.int64"]) +@pytest.mark.parametrize("ratios", [(1, 2)], ids=["(1, 2)"]) +def test_low_ratio_compression_metadata(loc_dtype, ratios): + rng = _seed(7) + for rows, nw in ((1, 1), (9, 9), (12, 5)): + seq_lens = torch.tensor( + [rng.choice([0, 1, 2, 3, 1000, 1001, 65535]) for _ in range(rows)], + dtype=torch.int32, + device=DEVICE, + ) + raw_out_loc = torch.randint(0, 1 << 24, (nw,), dtype=loc_dtype, device=DEVICE) + out = low_ratio_compression_metadata(seq_lens, raw_out_loc, ratios) + assert set(out) == { + f"c{r}_{k}" for r in ratios for k in ("out_loc", "topk_lengths_clamp1") + } + for r in ratios: + ref_loc, ref_clamp1 = _low_ratio_compression_metadata( + r, seq_lens, raw_out_loc + ) + assert out[f"c{r}_out_loc"].dtype is ref_loc.dtype + assert torch.equal(out[f"c{r}_out_loc"], ref_loc) + assert out[f"c{r}_topk_lengths_clamp1"].dtype is ref_clamp1.dtype + assert torch.equal(out[f"c{r}_topk_lengths_clamp1"], ref_clamp1) + + +def test_page_table_from_req_to_token_matches_torch(): + from sglang.kernels.ops.attention.dsv4.attn_glue_hip import ( + page_table_from_req_to_token, + ) + + _seed(23) + req_to_token = torch.randint( + 0, 2**20, (300, 8192), device=DEVICE, dtype=torch.int32 + ) + req_to_token[ + 7, :64 + ] = -3 # torch floor-divides; the slot values are never negative in serving + for bs, max_seq_len, page in ( + (1, 1000, 256), + (64, 8191, 256), + ): + req = torch.randint(0, 300, (bs,), device=DEVICE, dtype=torch.int32) + req[0] = 7 + ref = (req_to_token[req, :max_seq_len:page] // page).to(torch.int32) + got = page_table_from_req_to_token(req_to_token, req, max_seq_len, page) + assert got.shape == ref.shape and got.dtype == torch.int32 + assert torch.equal(got, ref) + empty = page_table_from_req_to_token( + req_to_token, torch.empty(0, device=DEVICE, dtype=torch.int32), 1000, 64 + ) + assert empty.shape == (0, 16) + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-v"])) diff --git a/test/registered/kernels/ops/attention/dsv4/test_swapab_attention.py b/test/registered/kernels/ops/attention/dsv4/test_swapab_attention.py new file mode 100644 index 0000000000..f9ab0a9bb4 --- /dev/null +++ b/test/registered/kernels/ops/attention/dsv4/test_swapab_attention.py @@ -0,0 +1,227 @@ +"""V4 packed-page addressing, softmax/sink and dynamic graph replay checks.""" + +import unittest + +import torch + +from sglang.kernels.ops.attention.dsv4.elementwise import fused_rope_inplace +from sglang.srt.utils import is_gfx95_supported, is_hip +from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=120, stage="base-b-kernel-unit", runner_config="1-gpu-large") +# backend-specific: the HIP swap-AB Gluon kernel runs on gfx950. +register_amd_ci(est_time=30, stage="stage-b", runner_config="1-gpu-small-amd-mi35x") + +if is_hip(): + from sglang.kernels.ops.attention.dsv4.swapab_gluon_hip import swapab_attention +else: + from sglang.kernels.ops.attention.dsv4.decode_attention_sm100 import ( + swapab_attention, + ) + + +def make_cache(page, pages=11): + # Page padding and poisoned unused bytes catch confusion between a view's + # logical token stride and the physically packed payload/scales planes. + stride = page * 584 + 256 + raw = torch.full((pages, stride), 0x7F, dtype=torch.uint8, device="cuda") + fp8 = torch.randn(pages, page, 448, device="cuda").to(torch.float8_e4m3fn) + exponents = torch.randint( + 124, 130, (pages, page, 7), device="cuda", dtype=torch.uint8 + ) + rope = torch.randn(pages, page, 64, device="cuda").to(torch.bfloat16) + raw[:, : page * 576].view(pages, page, 576)[..., :448].copy_(fp8.view(torch.uint8)) + raw[:, : page * 576].view(pages, page, 576)[..., 448:].copy_(rope.view(torch.uint8)) + raw[:, page * 576 : page * 584].view(pages, page, 8)[..., :7].copy_(exponents) + values = torch.cat( + ( + ( + fp8.float().view(pages, page, 7, 64) + * torch.exp2(exponents.float() - 127)[..., None] + ).flatten(-2), + rope.float(), + ), + -1, + ).reshape(-1, 512) + cache = raw.as_strided((pages, page, 1, 584), (stride, 584, 584, 1)) + return raw, cache, values + + +def reference(q, values, ids, lengths, sink, extra=None): + def gather(vals, idx, lens): + idx = idx[:, 0] + valid = (idx >= 0) & (idx < vals.shape[0]) + valid &= torch.arange(idx.shape[-1], device=idx.device)[None, :] < lens[:, None] + data = vals[idx.clamp(0, vals.shape[0] - 1).long()].double() + return torch.where(valid[..., None], data, 0), valid + + kv, valid = gather(values, ids, lengths) + if extra is not None: + ev, em = gather(*extra) + kv, valid = torch.cat((kv, ev), 1), torch.cat((valid, em), 1) + scores = q[:, 0].double() @ kv.transpose(1, 2) * 512**-0.5 + scores.masked_fill_(~valid[:, None, :], -torch.inf) + scores = torch.cat( + (scores, sink[None, :16, None].double().expand(q.shape[0], -1, -1)), -1 + ) + prob = torch.nan_to_num(torch.softmax(scores, -1)[..., :-1]) + return prob @ kv + + +@unittest.skipUnless( + torch.cuda.is_available() + and (is_gfx95_supported() or torch.cuda.get_device_capability()[0] == 10), + "requires Blackwell SM100/SM103 or AMD gfx950", +) +class TestSwapABAttention(CustomTestCase): + def check_case(self, b, page, nk, ne, graph=False): + torch.manual_seed(20260915 + b + nk + ne) + storage = torch.full( + (b, 1, 64, 512), float("nan"), device="cuda", dtype=torch.bfloat16 + ) + q = storage[:, :, :16] + q.copy_(torch.randn_like(q) * 0.5) + raw, kv, values = make_cache(page) + extra_raw, extra_kv, extra_values = make_cache(16) + # Strided rows exercise source views made by metadata alignment. + ids_storage = torch.randint( + -3, values.shape[0] + 4, (b, 1, nk + 19), device="cuda", dtype=torch.int32 + ) + ids = ids_storage[..., :nk] + ei_storage = torch.randint( + -3, + extra_values.shape[0] + 4, + (b, 1, ne + 17), + device="cuda", + dtype=torch.int32, + ) + ei = ei_storage[..., :ne] + lengths = torch.tensor( + [0, 1, 31, 63, 64, nk - 1, nk, nk + 3][:b], device="cuda", dtype=torch.int32 + ) + el = torch.tensor( + [ne, ne // 2, 1, 0, 63, ne - 1, 65, ne][:b], + device="cuda", + dtype=torch.int32, + ) + sink = torch.randn(64, device="cuda") + sink[0], sink[1], sink[2] = -torch.inf, torch.inf, 20.0 + originals = [x.clone() for x in (raw, extra_raw, ids, ei, q, lengths, el, sink)] + + def candidate(): + return swapab_attention( + q, + kv, + ids, + lengths, + sink, + extra_kv if ne else None, + ei if ne else None, + el if ne else None, + ) + + def check(out): + gold = reference( + q, values, ids, lengths, sink, (extra_values, ei, el) if ne else None + ) + floor_error = (gold.bfloat16().double() - gold).square().mean().sqrt() + error = (out.double() - gold).square().mean().sqrt() + self.assertLessEqual(error.item(), floor_error.item() * 1.01 + 1e-6) + # Absolute error is bounded by a BF16 output ULP plus a small + # accumulation allowance, including cancellation near zero. + torch.testing.assert_close(out.float(), gold.float(), atol=2e-5, rtol=0.004) + + out = candidate() + check(out) + for before, after in zip( + originals, (raw, extra_raw, ids, ei, q, lengths, el, sink) + ): + torch.testing.assert_close(before, after, rtol=0, atol=0, equal_nan=True) + if graph: + stream = torch.cuda.Stream() + stream.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(stream): + candidate() + torch.cuda.current_stream().wait_stream(stream) + g = torch.cuda.CUDAGraph() + with torch.cuda.graph(g, stream=stream): + replay_out = candidate() + for iteration in range(12): + # Changing device metadata and query values must affect replay; + # none of the lengths or sparse selections are host constants. + q.copy_(torch.randn_like(q) * (0.2 + iteration * 0.01)) + ids.copy_( + torch.randint( + -1, values.shape[0], ids.shape, device="cuda", dtype=torch.int32 + ) + ) + lengths.copy_( + torch.randint( + 0, nk + 1, lengths.shape, device="cuda", dtype=torch.int32 + ) + ) + if ne: + ei.copy_( + torch.randint( + -1, + extra_values.shape[0], + ei.shape, + device="cuda", + dtype=torch.int32, + ) + ) + el.copy_( + torch.randint( + 0, ne + 1, el.shape, device="cuda", dtype=torch.int32 + ) + ) + g.replay() + check(replay_out) + + @unittest.skipUnless(is_gfx95_supported(), "HIP fused inverse RoPE") + def test_fused_inverse_rope(self): + torch.manual_seed(103) + q = torch.randn(4, 16, 512, device="cuda", dtype=torch.bfloat16) * 0.25 + _, cache, _ = make_cache(32) + indices = torch.randint(352, (4, 128), device="cuda", dtype=torch.int32) + lengths = torch.full((4,), 128, device="cuda", dtype=torch.int32) + sink = torch.randn(16, device="cuda") + angles = torch.randn(16, 32, device="cuda") + freqs = torch.stack((angles.cos(), angles.sin()), dim=-1).flatten(1) + positions = torch.arange(4, device="cuda") + 7 + for extra in (False, True): + kwargs = ( + dict(extra_kv=cache, extra_indices=indices, extra_lengths=lengths) + if extra + else {} + ) + expected = swapab_attention(q, cache, indices, lengths, sink, **kwargs) + fused_rope_inplace( + expected[..., -64:], + None, + torch.view_as_complex(freqs.view(16, 32, 2)), + positions, + inverse=True, + ) + actual = swapab_attention( + q, cache, indices, lengths, sink, inv_rope=(freqs, positions), **kwargs + ) + torch.testing.assert_close(actual, expected, atol=0, rtol=0) + + def test_packed_pages_and_sink(self): + for case in ( + (1, 1, 1, 0), + (5, 16, 133, 0), + (8, 128, 192, 1024), + (6, 16, 192, 65), + ): + with self.subTest(case=case): + self.check_case(*case) + + def test_dynamic_graph_replay(self): + self.check_case(6, 64, 192, 512, graph=True) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/kernels/ops/attention/test_dsv4_q_rope_store.py b/test/registered/kernels/ops/attention/test_dsv4_q_rope_store.py index c4943112a8..3f34f84ccc 100644 --- a/test/registered/kernels/ops/attention/test_dsv4_q_rope_store.py +++ b/test/registered/kernels/ops/attention/test_dsv4_q_rope_store.py @@ -2,15 +2,25 @@ import unittest import torch +from sglang.kernels.ops.attention import deepseek_v4_rope from sglang.kernels.ops.attention.dsv4.elementwise import fused_rope_inplace from sglang.kernels.ops.attention.dsv4.q_rope_store import q_rope_store -from sglang.test.ci.ci_register import register_cuda_ci +from sglang.srt.utils import is_hip +from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci from sglang.test.test_utils import CustomTestCase register_cuda_ci(est_time=40, stage="base-b-kernel-unit", runner_config="1-gpu-large") +# gfx950: q_rope_store's HIP operation order matches the batched RoPE DeepseekV4 turns on +register_amd_ci(est_time=40, stage="stage-b", runner_config="1-gpu-small-amd-mi35x") class TestQRopeStore(CustomTestCase): + def setUp(self): + if is_hip(): + previous = deepseek_v4_rope._USE_BATCHED_ROPE + deepseek_v4_rope.set_batched_rope(True) + self.addCleanup(deepseek_v4_rope.set_batched_rope, previous) + def test_exact_output_and_padding(self): torch.manual_seed(911) freqs = torch.polar(