From e446698cdd09ffa9c6b464228d6d98db2a311be6 Mon Sep 17 00:00:00 2001 From: yunyang_nv Date: Tue, 6 Oct 2026 12:08:05 +0800 Subject: [PATCH] [SM120&90] Add CUDA fused Triton sparse-MLA prefill backend for DSA (#32779) Co-authored-by: Claude Fable 5 Co-authored-by: Po-Han Huang (NVIDIA) <53919306+nvpohanh@users.noreply.github.com> Co-authored-by: Po-Han Huang --- .../advanced_features/attention_backend.mdx | 6 + .../advanced_features/server_arguments.mdx | 8 +- .../dsa/triton_sparse_mla_prefill.py | 511 ++++++++++++++++++ .../kernels/ops/attention/flash_mla_sm120.py | 9 +- python/sglang/srt/arg_groups/fields/exec_.py | 11 + .../srt/layers/attention/dsa_backend.py | 140 ++++- .../attention_methods/dsa_attention.py | 10 +- .../bench_dsa_triton_sparse_mla_prefill.py | 193 +++++++ .../test_dsa_triton_sparse_mla_prefill.py | 266 +++++++++ .../test_triton_sparse_mla_prefill.py | 322 +++++++++++ .../unit/test_flashinfer_sparse_mla.py | 8 + 11 files changed, 1475 insertions(+), 9 deletions(-) create mode 100644 python/sglang/kernels/ops/attention/dsa/triton_sparse_mla_prefill.py create mode 100644 test/registered/kernels/benchmark/attention/bench_dsa_triton_sparse_mla_prefill.py create mode 100644 test/registered/kernels/ops/attention/test_dsa_triton_sparse_mla_prefill.py create mode 100644 test/registered/unit/attention/test_triton_sparse_mla_prefill.py diff --git a/docs/docs/advanced_features/attention_backend.mdx b/docs/docs/advanced_features/attention_backend.mdx index f8a60ff1f3..270a956b7b 100644 --- a/docs/docs/advanced_features/attention_backend.mdx +++ b/docs/docs/advanced_features/attention_backend.mdx @@ -438,6 +438,12 @@ Internally, the DSA backend dispatches to different sub-backends for prefill and ❌ Native FP8 (q8×kv8) sparse prefill on Hopper (SM90); requires --kv-cache-dtype fp8_e4m3 + + triton_sparse_mla + ✅ + ❌ + Fused Triton sparse prefill that works on the post-TP head count instead of padding heads to 64/128; opt-in. CUDA SM90+ (tiles tuned on SM90 and SM120; other architectures run an untuned tile and log a warning at startup), at most 32 query heads per rank, not with index_kpool. --dsa-triton-union 2|4 lets adjacent query tokens share one gathered index set (exact; num_q_heads * union must be a power of two in [16, 32]) + flashmla_kv ✅ diff --git a/docs/docs/advanced_features/server_arguments.mdx b/docs/docs/advanced_features/server_arguments.mdx index 533b1025a5..466b858bce 100644 --- a/docs/docs/advanced_features/server_arguments.mdx +++ b/docs/docs/advanced_features/server_arguments.mdx @@ -1470,7 +1470,13 @@ Combining `--enable-response-store` with `--disaggregation-mode=prefill` or `dec `--dsa-prefill-backend` DSA backend for the prefill stage (overrides `--attention-backend` when running DeepSeek DSA-style attention). Auto (hardware-dependent) - flashmla_sparse, flashmla_sparse_q8, flashmla_kv, flashmla_auto, fa3, tilelang, aiter, trtllm + flashmla_sparse, flashmla_sparse_q8, triton_sparse_mla, flashmla_kv, flashmla_auto, fa3, tilelang, aiter, trtllm + + + `--dsa-triton-union` + Union group size for `--dsa-prefill-backend triton_sparse_mla`: adjacent query tokens share one gathered index set (exact; an ownership mask restores each token's own softmax). Requires `num_q_heads * union` to be a power of two in [16, 32]; forced off under `--enable-deterministic-inference`. + `0` + 0, 2, 4 `--dsa-decode-backend` diff --git a/python/sglang/kernels/ops/attention/dsa/triton_sparse_mla_prefill.py b/python/sglang/kernels/ops/attention/dsa/triton_sparse_mla_prefill.py new file mode 100644 index 0000000000..7d52e5bb1f --- /dev/null +++ b/python/sglang/kernels/ops/attention/dsa/triton_sparse_mla_prefill.py @@ -0,0 +1,511 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the SGLang project +"""Fused Triton sparse-MLA prefill for DeepSeek Sparse Attention (DSA) on CUDA. + +One Triton program per query token: the heads are padded into a 16-row MMA tile +rather than into the grid, the online softmax stays in registers, and ``V`` is +read as ``kv[:, :d_v]``, so there are no partials and no merge pass. +``dsa/triton_sparse_mla.py`` is the gfx950-only FP8 sibling of this bf16 module. + +Interface matches ``sgl_kernel.flash_mla.flash_mla_sparse_fwd``:: + + q [T, H, 576] bf16 (absorbed MLA: 512 nope + 64 rope) + kv [S, 576] bf16 (V is kv[:, :512]) + indices [T, topk] int32 (-1 marks an invalid slot; no upper-bound check) + out [T, H, 512] bf16 + +A row of ``indices`` must not name the same KV position twice. Top-k selection +cannot, so this holds for every DSA caller; it is stated because ``union`` +gathers the distinct union of G rows and weights each position once, whereas +the base path would weight a repeat twice. + +``union`` (opt-in, 2 or 4): G adjacent query tokens share one gathered union +index set, and a per-row ownership bitmask restores each token's own softmax +support, so the result is the per-token result up to accumulation order. It +pays off because neighbouring tokens' selections overlap heavily on real indexer +output; the measurements are in the PR that added this module. + +All tunables are explicit arguments; the module reads no environment variables. +If a tuned tile exceeds the device shared-memory budget, the launcher steps down +through smaller tiles instead of raising ``OutOfResources``. +""" + +import logging + +import torch +import triton +import triton.language as tl + +logger = logging.getLogger(__name__) + +# Swept on the hardware named; anything else gets _UNTUNED_DEFAULT and a warning. +_PINNED = { + (9, 0): (64, 8, 2), # SM90, swept at T=8192 + (12, 0): (64, 4, 2), # SM120, swept at T=8192 + (12, 1): (64, 4, 2), # SM121: SM120's tile, not swept separately +} +_UNTUNED_DEFAULT = (64, 8, 3) +_UNTUNED_ARCH_WARNED = set() + + +@triton.jit +def _sparse_mla_prefill_kernel( + q_ptr, + kv_ptr, + idx_ptr, + len_ptr, + o_ptr, + sm_scale, + topk, + H: tl.constexpr, + BLOCK_H: tl.constexpr, + D_QK: tl.constexpr, + D_V: tl.constexpr, + BLOCK_N: tl.constexpr, + IDX64: tl.constexpr, +): + t = tl.program_id(0) + # Token offsets in int64: at h=128 the q offset t*H*D_QK passes 2^31 from + # t=29128 and the output offset from t=32768, wrapping to another token. + t64 = t.to(tl.int64) + D_TAIL: tl.constexpr = D_QK - D_V + + h = tl.arange(0, BLOCK_H) + hmask = h < H + dv = tl.arange(0, D_V) + dt = tl.arange(0, D_TAIL) + + qb = q_ptr + t64 * H * D_QK + q_main = tl.load( + qb + h[:, None] * D_QK + dv[None, :], mask=hmask[:, None], other=0.0 + ) + q_tail = tl.load( + qb + h[:, None] * D_QK + (D_V + dt)[None, :], mask=hmask[:, None], other=0.0 + ) + + m_i = tl.full([BLOCK_H], -float("inf"), tl.float32) + l_i = tl.zeros([BLOCK_H], tl.float32) + acc = tl.zeros([BLOCK_H, D_V], tl.float32) + + n = tl.arange(0, BLOCK_N) + k_len = tl.load(len_ptr + t) + for k0 in tl.range(0, k_len, BLOCK_N): + idx = tl.load(idx_ptr + t64 * topk + k0 + n, mask=(k0 + n) < k_len, other=-1) + valid = idx >= 0 + if IDX64: + row = tl.where(valid, idx, 0).to(tl.int64) + else: + row = tl.where(valid, idx, 0) + kb = kv_ptr + row[:, None] * D_QK + kv_main = tl.load(kb + dv[None, :], mask=valid[:, None], other=0.0) + kv_tail = tl.load(kb + (D_V + dt)[None, :], mask=valid[:, None], other=0.0) + + qk = tl.dot(q_main, tl.trans(kv_main)) + qk = tl.dot(q_tail, tl.trans(kv_tail), qk) * sm_scale + qk = tl.where(valid[None, :], qk, -float("inf")) + + m_new = tl.maximum(m_i, tl.max(qk, axis=1)) + m_safe = tl.where(m_new == -float("inf"), 0.0, m_new) + alpha = tl.exp(m_i - m_safe) + p = tl.exp(qk - m_safe[:, None]) + l_i = l_i * alpha + tl.sum(p, axis=1) + acc = acc * alpha[:, None] + tl.dot(p.to(kv_main.dtype), kv_main) + m_i = m_new + + l_safe = tl.where(l_i == 0.0, 1.0, l_i) + acc = acc * (1.0 / l_safe[:, None]) + tl.store( + o_ptr + t64 * H * D_V + h[:, None] * D_V + dv[None, :], + acc.to(o_ptr.dtype.element_ty), + mask=hmask[:, None], + ) + + +# --------------------------------------------------------------------------- +# Union path: G adjacent tokens share one gathered KV set. Per group, the G*K +# selected rows are sorted and deduplicated on the device, and an ownership bit +# per unique row lets the kernel below restore each token's own softmax +# support. The cost is O(G*K*log K) per group whatever the KV span, so +# nothing here depends on the index range, reads back to the host, or +# persists across calls. +# --------------------------------------------------------------------------- + + +@triton.jit +def _union_dedup_kernel( + sorted_ptr, + uidx_ptr, + ubits_ptr, + ulen_ptr, + K: tl.constexpr, + G: tl.constexpr, + LOG_K: tl.constexpr, +): + # One program per group of G tokens whose K selected rows are each sorted + # ascending (-1 pads sort first). Token i emits row v if no earlier token + # selected it; its ownership bits OR in every token that did. Membership is + # a vectorised lower_bound over the other token's list, which stays in L1. + g = tl.program_id(0).to(tl.int64) + grp = sorted_ptr + g * G * K + n = tl.arange(0, K) + cursor = tl.zeros([], tl.int32) + for i in tl.static_range(G): + v = tl.load(grp + i * K + n) + valid = v >= 0 + bits = tl.where(valid, 1 << i, 0).to(tl.int32) + first = valid + for j in tl.static_range(G): + if j != i: + lst = grp + j * K + lo = tl.zeros([K], tl.int32) + hi = tl.full([K], K, tl.int32) + for _ in tl.static_range(LOG_K): + mid = (lo + hi) // 2 + m = tl.load(lst + mid, mask=mid < K, other=2147483647) + lt = m < v + lo = tl.where(lt, mid + 1, lo) + hi = tl.where(lt, hi, mid) + found = (tl.load(lst + lo, mask=lo < K, other=-2) == v) & valid + bits |= tl.where(found, 1 << j, 0).to(tl.int32) + if j < i: + first = first & (found == 0) + wpos = cursor + tl.cumsum(first.to(tl.int32), axis=0) - first.to(tl.int32) + tl.store(uidx_ptr + g * (G * K) + wpos, v, mask=first) + tl.store(ubits_ptr + g * (G * K) + wpos, bits, mask=first) + cursor += tl.sum(first.to(tl.int32)) + tl.store(ulen_ptr + g, cursor) + + +# Per-group unique rows and ownership bits for [NG*G, K] indices: uidx [NG, G*K] +# int32 holds each group's distinct rows in its first ulen[g] slots (token-major, +# each token's new rows ascending, the rest unspecified), ubits [NG, G*K] int32 +# has bit i set when token i selected the row, ulen [NG] int32. torch sorts each +# K-wide row in shared memory; one Triton pass does the cross-token membership. +def _union_dedup(idx_main, G): + T_main, K = idx_main.shape + NG = T_main // G + srt = torch.sort(idx_main, dim=1)[0] + uidx = torch.empty(NG, G * K, dtype=torch.int32, device=idx_main.device) + ubits = torch.empty_like(uidx) + ulen = torch.empty(NG, dtype=torch.int32, device=idx_main.device) + _union_dedup_kernel[(NG,)]( + srt, + uidx, + ubits, + ulen, + K=K, + G=G, + # log2(K) + 1 halvings: the last one resolves a one-element interval. + LOG_K=K.bit_length(), + num_warps=16 if G == 4 else 8, # swept on SM90 at K=2048 + ) + return uidx, ubits, ulen + + +@triton.jit +def _sparse_mla_prefill_union_kernel( + q_ptr, + kv_ptr, + uidx_ptr, + ubits_ptr, + ulen_ptr, + o_ptr, + sm_scale, + U_STRIDE, + H: tl.constexpr, + G: tl.constexpr, + D_QK: tl.constexpr, + D_V: tl.constexpr, + BLOCK_N: tl.constexpr, +): + g = tl.program_id(0) + D_TAIL: tl.constexpr = D_QK - D_V + GH: tl.constexpr = G * H + + r = tl.arange(0, GH) + tok_of_row = r // H + dv = tl.arange(0, D_V) + dt = tl.arange(0, D_TAIL) + + qb = q_ptr + g.to(tl.int64) * GH * D_QK + q_main = tl.load(qb + r[:, None] * D_QK + dv[None, :]) + q_tail = tl.load(qb + r[:, None] * D_QK + (D_V + dt)[None, :]) + + m_i = tl.full([GH], -float("inf"), tl.float32) + l_i = tl.zeros([GH], tl.float32) + acc = tl.zeros([GH, D_V], tl.float32) + + n = tl.arange(0, BLOCK_N) + u_len = tl.load(ulen_ptr + g) + ub = g.to(tl.int64) * U_STRIDE + for k0 in tl.range(0, u_len, BLOCK_N): + inb = (k0 + n) < u_len + uidx = tl.load(uidx_ptr + ub + k0 + n, mask=inb, other=-1) + bits = tl.load(ubits_ptr + ub + k0 + n, mask=inb, other=0) + valid = uidx >= 0 + row = tl.where(valid, uidx, 0).to(tl.int64) + kb = kv_ptr + row * D_QK + kv_main = tl.load(kb[:, None] + dv[None, :], mask=valid[:, None], other=0.0) + kv_tail = tl.load( + kb[:, None] + (D_V + dt)[None, :], mask=valid[:, None], other=0.0 + ) + + qk = tl.dot(q_main, tl.trans(kv_main)) + qk = tl.dot(q_tail, tl.trans(kv_tail), qk) * sm_scale + sel = ((bits[None, :] >> tok_of_row[:, None]) & 1) != 0 + qk = tl.where(sel & valid[None, :], qk, -float("inf")) + + m_new = tl.maximum(m_i, tl.max(qk, axis=1)) + m_safe = tl.where(m_new == -float("inf"), 0.0, m_new) + alpha = tl.exp(m_i - m_safe) + p = tl.exp(qk - m_safe[:, None]) + l_i = l_i * alpha + tl.sum(p, axis=1) + acc = acc * alpha[:, None] + tl.dot(p.to(kv_main.dtype), kv_main) + m_i = m_new + + l_safe = tl.where(l_i == 0.0, 1.0, l_i) + acc = acc / l_safe[:, None] + tl.store( + o_ptr + g.to(tl.int64) * GH * D_V + r[:, None] * D_V + dv[None, :], + acc.to(o_ptr.dtype.element_ty), + ) + + +# Returns True if handled; the T % G tail rows go through the per-token path. +def _union_path(*, q, kv, indices, sm_scale, d_v, out, G, union_config): + T, h, d_qk = q.shape + K = indices.shape[-1] + rows = G * h + # tl.arange needs a power of two and tl.dot needs M >= 16; the tuned tiles + # stop at 32 rows. The backend validator rejects the same shapes at startup. + if G < 2 or rows < 16 or rows > 32 or rows & (rows - 1): + return False + if K & (K - 1): # the dedup kernel tiles each token's list with tl.arange(0, K) + return False + T_main = (T // G) * G + if T_main == 0: + return False + NG = T_main // G + uidx, ubits, ulen = _union_dedup(idx_main=indices[:T_main], G=G) + U_STRIDE = G * K + if union_config is not None: + bn, warps, stages = union_config + elif torch.cuda.get_device_capability(q.device)[0] >= 12: + # SM120 sweeps on real indices: (64,4,3) at G=2. At G=4 the 32-row Q + # tile makes BN=64 exceed the 100 KB budget; (32,4,2) beats every neighbour. + bn, warps, stages = (64, 4, 3) if G == 2 else (32, 4, 2) + else: + bn, warps, stages = (64, 4, 2) if G == 4 else (64, 8, 2) + # The union Q tile is G*H rows, so its shared-memory footprint grows with + # the head count (16 heads at G=2 exceeds SM120's 100 KB with the tuned + # tile). Step down like the per-token launcher rather than fail the request. + for bn_try, ns_try in _tile_candidates("union", h, G, q.device, bn, stages): + try: + _sparse_mla_prefill_union_kernel[(NG,)]( + q[:T_main], + kv, + uidx, + ubits, + ulen, + out[:T_main], + sm_scale, + U_STRIDE, + H=h, + G=G, + D_QK=d_qk, + D_V=d_v, + BLOCK_N=bn_try, + num_warps=warps, + num_stages=ns_try, + ) + except triton.runtime.errors.OutOfResources: + continue + _FIT_TILE[("union", h, G, q.device.index)] = (bn_try, ns_try) + break + else: + return False # no tile fits; caller falls through to the per-token path + if T_main < T: + sparse_mla_prefill( + q[T_main:], + kv, + indices[T_main:], + sm_scale, + d_v, + out=out[T_main:], + union=0, + ) + return True + + +def _topk_length(indices, topk): + valid = indices >= 0 + any_valid = valid.any(dim=-1) + last = topk - torch.flip(valid, [-1]).int().argmax(dim=-1) + return torch.where(any_valid, last, torch.zeros_like(last)).to(torch.int32) + + +# Per-arch tuned (BLOCK_N, num_warps, num_stages); see _PINNED. +def _config(device): + cap = torch.cuda.get_device_capability(device) + if cap in _PINNED: + return _PINNED[cap] + if cap not in _UNTUNED_ARCH_WARNED: + _UNTUNED_ARCH_WARNED.add(cap) + # The kernel is correct on any SM90+ device, but the tile was only swept + # on the architectures in _PINNED. Say so rather than quietly running a + # config nobody has measured; add an entry there once swept. + logger.warning( + "triton_sparse_mla: no tuned tile for sm_%d%d; falling back to %s. " + "Sweep and add it to _PINNED for best throughput.", + cap[0], + cap[1], + _UNTUNED_DEFAULT, + ) + return _UNTUNED_DEFAULT + + +# Ordered (BLOCK_N, num_stages) candidates: the tuned config first, then +# progressively smaller shared-memory footprints, so one pinned config serves +# head counts and devices whose budget the tuned tile would exceed. +def _smem_fallbacks(bn, stages): + seen, out = set(), [] + for cand in ( + (bn, stages), + (bn, 2), + (bn // 2, stages), + (bn // 2, 2), + (bn // 4, 2), + (16, 2), + ): + b, ns = max(16, cand[0]), max(1, cand[1]) + if (b, ns) not in seen: + seen.add((b, ns)) + out.append((b, ns)) + return out + + +# The tile that fit last time, per (path, num_heads, group, device index). The +# tuned tile can exceed a device's shared memory (h=32 on SM120's 100 KB); the +# step-down must then run once per shape, not on every layer of every prefill. +_FIT_TILE = {} + + +# _smem_fallbacks with the tile that fit last time moved to the front. +def _tile_candidates(path, h, G, device, bn, stages): + cands = _smem_fallbacks(bn, stages) + fit = _FIT_TILE.get((path, h, G, device.index)) + if fit in cands: + cands.remove(fit) + cands.insert(0, fit) + return cands + + +def sparse_mla_prefill( + q, + kv, + indices, + sm_scale, + d_v=512, + *, + topk_length=None, + out=None, + union=0, + config=None, + union_config=None, + int64_indexing=None, +): + """Fused sparse-MLA prefill. Returns ``out`` ``[T, H, d_v]`` bf16. + + Args: + q: ``[T, H, d_qk]`` bf16 query (absorbed MLA; ``d_qk = d_v + rope``). + kv: ``[S, d_qk]`` or ``[S, 1, d_qk]`` bf16 latent cache; ``V`` is + ``kv[:, :d_v]`` (no separate value gather). + indices: ``[T, topk]`` or ``[T, 1, topk]`` int32 selected slots; ``-1`` + marks an invalid slot and is skipped. Every other value must be a + row of ``kv``: the kernels make no upper-bound check. + sm_scale: softmax scale. + d_v: value head dim (512 for DSA). + topk_length: optional ``[T]`` int32 per-row valid count. Computed from + ``indices`` when omitted; pass it to skip that reduction. + out: optional preallocated ``[T, H, d_v]`` bf16 output. + union: 0 (off), 2 or 4: share one gathered union index set across ``G`` + adjacent query tokens. Exact, not an approximation: an ownership + bitmask restores each token's own softmax support. + config / union_config: optional tile overrides + ``(BLOCK_N, num_warps, num_stages)``. Defaults are the per-arch + tuned entries in ``_PINNED``. + """ + if kv.dim() == 3: # [S, 1, D] -> [S, D] + assert kv.shape[1] == 1 + kv = kv.squeeze(1) + if indices.dim() == 3: # [T, 1, K] -> [T, K] + assert indices.shape[1] == 1 + indices = indices.squeeze(1) + T, h, d_qk = q.shape + topk = indices.shape[-1] + q, kv, indices = q.contiguous(), kv.contiguous(), indices.contiguous() + if kv.dtype != torch.bfloat16: + # An FP8 pool that reaches this entry (the PAGED transform leaves it + # packed) would otherwise fail in Triton compile or read fp8 bytes at a + # 576-element stride. + raise TypeError( + f"sparse_mla_prefill needs a bf16 KV cache, got {kv.dtype}; " + "dequantize the pool first." + ) + if out is None: + out = torch.empty(T, h, d_v, dtype=torch.bfloat16, device=q.device) + + if union in (2, 4) and _union_path( + q=q, + kv=kv, + indices=indices, + sm_scale=sm_scale, + d_v=d_v, + out=out, + G=union, + union_config=union_config, + ): + return out + + if topk_length is None: + topk_length = _topk_length(indices, topk) + + bn, warps, stages = config or _config(q.device) + # int32 gather addressing keeps the gather loop on IMAD; int64 only when the + # pool could overflow int32 element offsets (row*d_qk + d_qk-1), i.e. above + # ~3.7M rows at d_qk=576. No test can allocate that pool, so the mode is + # overridable to keep the int64 path reachable from a test. + if int64_indexing is None: + idx64 = kv.shape[0] > (2**31 - 1 - (d_qk - 1)) // d_qk + else: + idx64 = bool(int64_indexing) + block_h = max(16, triton.next_power_of_2(h)) + for bn_try, ns_try in _tile_candidates("base", h, 0, q.device, bn, stages): + try: + _sparse_mla_prefill_kernel[(T,)]( + q, + kv, + indices, + topk_length, + out, + sm_scale, + topk, + H=h, + BLOCK_H=block_h, + D_QK=d_qk, + D_V=d_v, + BLOCK_N=bn_try, + num_warps=warps, + num_stages=ns_try, + IDX64=idx64, + ) + except triton.runtime.errors.OutOfResources: + # A larger head count (BLOCK_H) or a smaller smem budget can push the + # pinned tile over the device limit (e.g. h=32 on SM120's 100 KB). + # Step down the K tile / pipeline depth instead of failing the request. + continue + _FIT_TILE[("base", h, 0, q.device.index)] = (bn_try, ns_try) + return out + raise triton.runtime.errors.OutOfResources( + 0, 0, "shared memory: no fallback config fits this device/shape" + ) diff --git a/python/sglang/kernels/ops/attention/flash_mla_sm120.py b/python/sglang/kernels/ops/attention/flash_mla_sm120.py index 5267908eee..c57194d3c5 100644 --- a/python/sglang/kernels/ops/attention/flash_mla_sm120.py +++ b/python/sglang/kernels/ops/attention/flash_mla_sm120.py @@ -700,12 +700,15 @@ def _validate_flashinfer_sparse_mla_backend( f"decode_impl={decode_impl!r}." ) if is_glm_sm12_fp8: - unsupported = selected - {"flashinfer_sparse_mla"} + # flashinfer_sparse_mla stays the auto-selected default on this + # platform; triton_sparse_mla is a validated alternative prefill + # implementation that the user may select explicitly. + unsupported = selected - {"flashinfer_sparse_mla", "triton_sparse_mla"} if unsupported: raise ValueError( "GLM DSA with FP8 KV cache on NVIDIA SM120/SM121 supports " - "only flashinfer_sparse_mla, " - f"but got {sorted(unsupported)}." + "only flashinfer_sparse_mla (default) or triton_sparse_mla " + f"(prefill), but got {sorted(unsupported)}." ) return uses_flashinfer_sparse_mla diff --git a/python/sglang/srt/arg_groups/fields/exec_.py b/python/sglang/srt/arg_groups/fields/exec_.py index aca25dcb88..6ff90b7ec4 100644 --- a/python/sglang/srt/arg_groups/fields/exec_.py +++ b/python/sglang/srt/arg_groups/fields/exec_.py @@ -217,6 +217,7 @@ class ExecKernel(msgspec.Struct): "flashmla_kv", "flashmla_auto", "flashinfer_sparse_mla", + "triton_sparse_mla", "fa3", "tilelang", "triton", @@ -237,6 +238,16 @@ class ExecKernel(msgspec.Struct): choices=["auto", "flashmla_sparse", "flashmla_sparse_q8"], ), ] = "auto" + dsa_triton_union: A[ + int, + Arg( + help="Union group size for --dsa-prefill-backend triton_sparse_mla: " + "0 disables, 2 or 4 make G adjacent query tokens share one gathered " + "index set (mathematically exact; an ownership mask restores each " + "token's own softmax support).", + choices=[0, 2, 4], + ), + ] = 0 dsa_decode_backend: A[ Optional[str], Arg( diff --git a/python/sglang/srt/layers/attention/dsa_backend.py b/python/sglang/srt/layers/attention/dsa_backend.py index bb05d9d59c..4794c46022 100644 --- a/python/sglang/srt/layers/attention/dsa_backend.py +++ b/python/sglang/srt/layers/attention/dsa_backend.py @@ -330,6 +330,7 @@ _DSA_IMPL_T: TypeAlias = Literal[ "flashmla_sparse_q8", "flashmla_kv", "flashinfer_sparse_mla", + "triton_sparse_mla", "fa3", "tilelang", "triton", @@ -338,6 +339,71 @@ _DSA_IMPL_T: TypeAlias = Literal[ ] +# Largest head count the kernel has been measured at. Above it BLOCK_H=64/128 +# makes the [BLOCK_H, 512] fp32 accumulator spill or miss SM120's smem budget. +_TRITON_SPARSE_MLA_MAX_HEADS = 32 + + +def _validate_triton_sparse_mla_backend( + *, + device_sm_major: int, + num_q_heads: int, + union: int, + index_kpool: int, +) -> None: + if _is_hip: + raise ValueError( + "triton_sparse_mla is a CUDA prefill backend; on ROCm use the " + "existing tilelang / aiter DSA paths." + ) + if device_sm_major < 9: + raise ValueError( + "triton_sparse_mla requires SM90 (Hopper) or newer; " + f"got sm_major={device_sm_major}." + ) + if num_q_heads > _TRITON_SPARSE_MLA_MAX_HEADS: + raise ValueError( + "triton_sparse_mla is validated up to " + f"{_TRITON_SPARSE_MLA_MAX_HEADS} query heads per rank; got " + f"num_q_heads={num_q_heads}. Raise the attention TP degree or use " + "another DSA prefill backend." + ) + if index_kpool > 1: + raise ValueError( + "triton_sparse_mla does not support index_kpool > 1 (the pooled " + f"indexer appends tail tokens to topk_indices); got {index_kpool}. " + "Use flashmla_sparse, which reroutes the tail." + ) + if union not in (0, 2, 4): + raise ValueError(f"--dsa-triton-union must be 0, 2 or 4; got {union}.") + if union: + # tl.arange needs a power of two and tl.dot needs M >= 16; the tuned + # union tiles stop at 32 rows. + rows = num_q_heads * union + if rows < 16 or rows > 32 or rows & (rows - 1): + raise ValueError( + f"--dsa-triton-union {union} needs num_q_heads * union to be a " + f"power of two in [16, 32] (the union mma tile); got " + f"num_q_heads={num_q_heads}, i.e. {rows} rows. Use a different " + "group size or disable union." + ) + + +def _resolve_dsa_triton_union(*, union: int, deterministic: bool) -> int: + # Union makes a token's output depend on which tokens share its group and + # on the T % G tail, which breaks the batch invariance deterministic + # inference promises. + if union and deterministic: + logger.warning( + "--dsa-triton-union %d is disabled under " + "--enable-deterministic-inference: the union path makes a token's " + "output depend on its batch neighbours.", + union, + ) + return 0 + return union + + class DeepseekSparseAttnBackend( DSAMetadataManagementMixin, DeepseekSparseAttnBackendKPoolMixin, @@ -407,6 +473,11 @@ class DeepseekSparseAttnBackend( self.supports_mha_one_shot: bool = True self.dsa_prefill_impl: _DSA_IMPL_T = get_exec().kernel.dsa_prefill_backend self.dsa_decode_impl: _DSA_IMPL_T = get_exec().kernel.dsa_decode_backend + # Opt-in exact fast path for the Triton sparse-MLA prefill kernel. + self.dsa_triton_union: int = _resolve_dsa_triton_union( + union=get_exec().kernel.dsa_triton_union, + deterministic=get_exec().deterministic.enable_deterministic_inference, + ) self.dsa_topk_backend: DSATopKBackend = DSATopKBackend.resolve(model_runner) if self.num_q_heads <= 64: self.flashmla_kv_num_q_heads = 64 @@ -514,6 +585,14 @@ class DeepseekSparseAttnBackend( f"capability sm_{self.device_sm_major}x." ) + if self.dsa_prefill_impl == "triton_sparse_mla": + _validate_triton_sparse_mla_backend( + device_sm_major=self.device_sm_major, + num_q_heads=self.num_q_heads, + union=self.dsa_triton_union, + index_kpool=self.dsa_index_kpool, + ) + # `flashmla_sparse_q8` is prefill-only (FP8 decode goes through # `flashmla_kv`); reject it as a decode backend, since argparse accepts it # via the shared CLI choices. @@ -2128,7 +2207,11 @@ class DeepseekSparseAttnBackend( else None ), ) - elif dsa_impl in ("flashmla_sparse", "flashmla_sparse_q8"): + elif dsa_impl in ( + "flashmla_sparse", + "flashmla_sparse_q8", + "triton_sparse_mla", + ): if topk_transform_method == TopkTransformMethod.RAGGED: _has_prefix = any(forward_batch.extend_prefix_lens_cpu) page_table_1 = topk_indices @@ -2185,7 +2268,9 @@ class DeepseekSparseAttnBackend( layer_id=layer.layer_id, ) - # bf16 path (dsa_impl == "flashmla_sparse"). + # bf16 path: `flashmla_sparse` and `triton_sparse_mla` take the + # same inputs (concatenated q, dequantized bf16 KV, top-k list + # as the slot table) and differ only in the kernel they call. if _has_prefix: page_table_1_flattened = ( self.forward_metadata.page_table_1_flattened @@ -2199,6 +2284,15 @@ class DeepseekSparseAttnBackend( if q_rope is not None: q_all = concat_mla_absorb_q_general(q_nope, q_rope) + if dsa_impl == "triton_sparse_mla": + return self._forward_triton_sparse_mla( + q_all=q_all, + kv_cache=kv_cache, + page_table_1=page_table_1, + sm_scale=layer.scaling, + v_head_dim=layer.v_head_dim, + topk_length=metadata.dsa_cache_seqlens_int32, + ) return self._forward_flashmla_sparse( q_all=q_all, kv_cache=kv_cache, @@ -2903,6 +2997,41 @@ class DeepseekSparseAttnBackend( o = o[:, :num_heads, :] return o + def _forward_triton_sparse_mla( + self, + q_all: torch.Tensor, + kv_cache: torch.Tensor, + v_head_dim: int, + page_table_1: torch.Tensor, + sm_scale: float, + topk_length: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + from sglang.kernels.ops.attention.dsa.triton_sparse_mla_prefill import ( + sparse_mla_prefill, + ) + + # Only the stream's own capture state gates union: the breakable prefill + # graph runs attention eagerly between its segments while + # get_is_capture_mode() stays set for the whole replay. Under real capture + # the per-token path runs instead (same result); union is untested there. + capturing = torch.cuda.is_current_stream_capturing() + union = 0 if capturing else self.dsa_triton_union + + # Same contract as `_forward_flashmla_sparse`: metadata rows are expected + # to match q rows; fall back to full-width compute if they ever diverge. + if topk_length is not None and topk_length.shape[0] != q_all.shape[0]: + topk_length = None + + return sparse_mla_prefill( + q_all, + kv_cache, + page_table_1, + sm_scale, + v_head_dim, + topk_length=topk_length, + union=union, + ) + def _forward_flashinfer_sparse_mla( self, q_all: torch.Tensor, @@ -3672,8 +3801,11 @@ class DeepseekSparseAttnBackend( # flashmla_sparse_q8 shares flashmla_sparse's RAGGED prefill routing — the q8 # dispatch lives inside the RAGGED branch of forward_extend; without this the # transform is PAGED, the q8 path is skipped, and the bf16 kernel crashes on - # fp8 KV ("kv must have dtype kBFloat16"). - and self.dsa_prefill_impl in ("flashmla_sparse", "flashmla_sparse_q8") + # fp8 KV ("kv must have dtype kBFloat16"). triton_sparse_mla is listed for + # the same reason: its dispatch also dequantizes inside the RAGGED branch + # and its kernel is bf16-only. + and self.dsa_prefill_impl + in ("flashmla_sparse", "flashmla_sparse_q8", "triton_sparse_mla") and forward_mode == ForwardMode.EXTEND ): topk_transform_method = TopkTransformMethod.RAGGED diff --git a/python/sglang/test/kits/attention_unittest/attention_methods/dsa_attention.py b/python/sglang/test/kits/attention_unittest/attention_methods/dsa_attention.py index 336c23698a..faebbeaeed 100644 --- a/python/sglang/test/kits/attention_unittest/attention_methods/dsa_attention.py +++ b/python/sglang/test/kits/attention_unittest/attention_methods/dsa_attention.py @@ -1209,6 +1209,13 @@ def dsa_impl_capability(impl: str) -> tuple[bool, str]: return False, f"aiter unavailable: {exc}" return True, "" + if impl == "triton_sparse_mla": + if is_hip(): + return False, "triton_sparse_mla is a CUDA-only prefill backend" + if major < 9: + return False, f"{impl} requires SM>=9.0, got SM{major}.x" + return True, "" + if impl == "flashmla_auto": # `flashmla_auto` resolves to flashmla_sparse / flashmla_kv at # forward time depending on `dsa_kv_cache_store_fp8`; both leaf @@ -1227,6 +1234,7 @@ DSA_PREFILL_IMPL_VARIANTS: tuple[str, ...] = ( "tilelang", "trtllm", "aiter", + "triton_sparse_mla", ) DSA_DECODE_IMPL_VARIANTS: tuple[str, ...] = ( "flashmla_sparse", @@ -1243,7 +1251,7 @@ DSA_DECODE_IMPL_VARIANTS: tuple[str, ...] = ( # take in FP8 deployments. The `flashmla_kv` decode kernel and *both* # flashmla prefill kernels are the production-relevant FP8 paths. DSA_FP8_COMPATIBLE_PREFILL_IMPLS: frozenset[str] = frozenset( - {"flashmla_sparse", "flashmla_kv", "flashmla_auto"} + {"flashmla_sparse", "flashmla_kv", "flashmla_auto", "triton_sparse_mla"} ) DSA_FP8_COMPATIBLE_DECODE_IMPLS: frozenset[str] = frozenset( {"flashmla_kv", "flashmla_auto"} diff --git a/test/registered/kernels/benchmark/attention/bench_dsa_triton_sparse_mla_prefill.py b/test/registered/kernels/benchmark/attention/bench_dsa_triton_sparse_mla_prefill.py new file mode 100644 index 0000000000..00c5cdda51 --- /dev/null +++ b/test/registered/kernels/benchmark/attention/bench_dsa_triton_sparse_mla_prefill.py @@ -0,0 +1,193 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Benchmark the fused Triton sparse-MLA prefill against the kernels the DSA +prefill can otherwise dispatch to. Indices are a calibrated random walk (see +``_make_inputs``): uniform-random indices lack the token-to-token overlap the +``union`` path depends on and understate it by about 2x. +""" + +from __future__ import annotations + +import torch +import triton +import triton.testing + +from sglang.kernels.jit.benchmark.utils import run_benchmark_no_cudagraph +from sglang.kernels.ops.attention.dsa.triton_sparse_mla_prefill import ( + sparse_mla_prefill, +) +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.utils import is_in_ci + +try: + from sgl_kernel.flash_mla import flash_mla_sparse_fwd +except ImportError: + flash_mla_sparse_fwd = None + +try: + # Same form as the kernel under test, but reachable only on gfx950. Kept as + # a provider so the comparison against the closest sibling is visible here + # rather than having to be taken on trust; it is dtype-generic enough to run + # in bf16 on CUDA for the purpose of timing. + from sglang.kernels.ops.attention.dsa.triton_sparse_mla import ( + triton_sparse_mla_fwd, + ) +except ImportError: + triton_sparse_mla_fwd = None + + +def _flashmla_runnable() -> bool: + # The op refuses anything but SM90a / SM100f at call time, so gate on the + # device rather than on the import succeeding. + return flash_mla_sparse_fwd is not None and ( + torch.cuda.is_available() and torch.cuda.get_device_capability()[0] in (9, 10) + ) + + +HAS_FLASHMLA = _flashmla_runnable() + +register_cuda_ci( + est_time=90, stage="base-b-kernel-benchmark", runner_config="1-gpu-large" +) + +IS_CI = is_in_ci() +D_QK, D_V, SM_SCALE = 576, 512, 0.0625 +CHURN = 0.02 # share of a token's rows replaced when moving to the next token +UNION_TILE_ROWS = 32 + +if IS_CI: + CASES = [(1024, 8, 512), (2048, 8, 2048)] +else: + # (num_tokens, num_q_heads_after_tp, topk) + CASES = [ + (2048, 8, 2048), + (4096, 8, 2048), + (8192, 8, 2048), + (8192, 16, 2048), + ] + +LINE_VALS = ["triton_prefill", "triton_prefill_union"] +LINE_NAMES = ["Triton prefill", "Triton prefill (union)"] +STYLES = [("blue", "-"), ("green", "-")] +if triton_sparse_mla_fwd is not None: + LINE_VALS.insert(0, "triton_sparse_mla_gfx950") + LINE_NAMES.insert(0, "Triton sparse-MLA (gfx950 path)") + STYLES.insert(0, ("red", ":")) +if HAS_FLASHMLA: + LINE_VALS.insert(0, "flashmla_sparse") + LINE_NAMES.insert(0, "FlashMLA sparse") + STYLES.insert(0, ("orange", "--")) + + +def _make_inputs(num_tokens: int, num_heads: int, topk: int): + generator = torch.Generator(device="cuda") + generator.manual_seed(7000 + num_tokens + num_heads) + # The KV pool must be comfortably larger than topk, or there are not enough + # unselected rows left to vary the set from token to token. + s_kv = max(num_tokens, 2 * topk) + q = torch.randn( + (num_tokens, num_heads, D_QK), + dtype=torch.bfloat16, + device="cuda", + generator=generator, + ) + kv = torch.randn( + (s_kv, D_QK), dtype=torch.bfloat16, device="cuda", generator=generator + ) + + # Random walk, not i.i.d. draws from a shared pool: the indexer's ranking + # drifts slowly with t, so token t+1's selection is token t's with a few + # rows swapped. CHURN is set so the walk reproduces the union sizes measured + # on captured GLM-5.1 traces: 1.00-1.12x one token's set for 2 adjacent + # tokens and 1.01-1.23x for 4, shape dependent. + perm = torch.randperm(s_kv, device="cuda", generator=generator) + current, spare = perm[:topk].clone(), perm[topk:].clone() + n_swap = max(1, int(topk * CHURN)) + assert spare.numel() >= n_swap, "KV pool too small to vary the selection" + indices = torch.empty((num_tokens, topk), dtype=torch.int32, device="cuda") + for t in range(num_tokens): + indices[t] = current.to(torch.int32) + if t < topk: + # A token whose causal prefix is shorter than topk cannot fill the + # row; the indexer pads with -1. Production prefill spends its first + # topk tokens here, so leaving it out would flatter every kernel + # that ignores the padding and penalise the ones that skip it. + indices[t, t + 1 :] = -1 + out_slots = torch.randperm(topk, device="cuda", generator=generator)[:n_swap] + in_slots = torch.randperm(spare.numel(), device="cuda", generator=generator)[ + :n_swap + ] + evicted = current[out_slots].clone() + current[out_slots] = spare[in_slots] + spare[in_slots] = evicted # swap keeps both sets duplicate-free + return q, kv, indices + + +@triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["num_tokens", "num_heads", "topk"], + x_vals=CASES, + line_arg="provider", + line_vals=LINE_VALS, + line_names=LINE_NAMES, + styles=STYLES, + ylabel="us", + plot_name="dsa-triton-sparse-mla-prefill-performance", + args={}, + ) +) +def bench_dsa_triton_sparse_mla_prefill( + num_tokens: int, num_heads: int, topk: int, provider: str +): + q, kv, indices = _make_inputs(num_tokens, num_heads, topk) + + if provider == "flashmla_sparse": + if not HAS_FLASHMLA: + raise RuntimeError("sgl_kernel.flash_mla.flash_mla_sparse_fwd unavailable") + kv_3d = kv.unsqueeze(1) + idx_3d = indices.unsqueeze(1) + # The kernel requires num_heads % 64 on SM90 and % 128 on SM100+, so the + # backend zero-pads the head dim and slices the result back. Do the same + # here: that padding is the cost being measured, not an artefact of the + # benchmark. + required = 128 if torch.cuda.get_device_capability()[0] >= 10 else 64 + if num_heads % required: + q_in = q.new_zeros((num_tokens, required, D_QK)) + q_in[:, :num_heads, :] = q + else: + q_in = q + + def fn(): + return flash_mla_sparse_fwd(q_in, kv_3d, idx_3d, SM_SCALE, D_V)[0][ + :, :num_heads, : + ] + + elif provider == "triton_sparse_mla_gfx950": + q_nope = q[:, :, :D_V].contiguous() + q_rope = q[:, :, D_V:].contiguous() + + def fn(): + return triton_sparse_mla_fwd(q_nope, q_rope, kv, indices, SM_SCALE, D_V) + + elif provider == "triton_prefill": + + def fn(): + return sparse_mla_prefill(q, kv, indices, SM_SCALE, D_V) + + elif provider == "triton_prefill_union": + # num_heads * group must fit the 32-row union tile, the same rule the + # backend validator applies; take the largest group that does. + group = next((g for g in (4, 2) if num_heads * g <= UNION_TILE_ROWS), 0) + if group == 0: + raise RuntimeError(f"no legal union group for num_heads={num_heads}") + + def fn(): + return sparse_mla_prefill(q, kv, indices, SM_SCALE, D_V, union=group) + + else: + raise ValueError(f"Unknown provider: {provider}") + + return run_benchmark_no_cudagraph(fn) + + +if __name__ == "__main__": + bench_dsa_triton_sparse_mla_prefill.run(print_data=True) diff --git a/test/registered/kernels/ops/attention/test_dsa_triton_sparse_mla_prefill.py b/test/registered/kernels/ops/attention/test_dsa_triton_sparse_mla_prefill.py new file mode 100644 index 0000000000..f37f0cd6a1 --- /dev/null +++ b/test/registered/kernels/ops/attention/test_dsa_triton_sparse_mla_prefill.py @@ -0,0 +1,266 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Numerical correctness of the fused Triton sparse-MLA prefill kernel against an +fp32 reference: base path, ragged ``-1`` padding, the union path's ownership mask +and dedup contract, int64 addressing, determinism, and the shared-memory step-down. +""" + +import unittest + +import torch + +from sglang.kernels.ops.attention.dsa.triton_sparse_mla_prefill import ( + _FIT_TILE, + _union_dedup, + sparse_mla_prefill, +) +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=15, stage="base-b-kernel-unit", runner_config="1-gpu-large") + +D_QK, D_V, SM_SCALE = 576, 512, 0.0625 + + +def _reference(q, kv, indices, d_v=D_V): + """fp32 reference: each token attends over its own valid selected rows.""" + T, h, _ = q.shape + S = kv.shape[0] + out = torch.empty(T, h, d_v, dtype=torch.float32, device=q.device) + qf, kf = q.float(), kv.float() + for t in range(T): + idx = indices[t] + idx = idx[(idx >= 0) & (idx < S)] + if idx.numel() == 0: + out[t] = 0.0 + continue + k = kf[idx] + p = torch.softmax((qf[t] @ k.T) * SM_SCALE, dim=-1) + out[t] = p @ k[:, :d_v] + return out.to(torch.bfloat16) + + +def _qkv(T, S, h, seed): + g = torch.Generator(device="cuda").manual_seed(seed) + q = torch.randn(T, h, D_QK, dtype=torch.bfloat16, device="cuda", generator=g) + kv = torch.randn(S, D_QK, dtype=torch.bfloat16, device="cuda", generator=g) + return q, kv, g + + +def _random_indices(T, topk, S, g, pad_frac=0.0): + idx = torch.full((T, topk), -1, dtype=torch.int32, device="cuda") + n = max(1, int(topk * (1.0 - pad_frac))) + for t in range(T): + idx[t, :n] = torch.randperm(S, device="cuda", generator=g)[:n].to(torch.int32) + return idx + + +def _overlapping_indices(T, topk, S, g): + """Selections that mostly agree between neighbouring tokens, as the indexer's + do; uniform-random sets are nearly disjoint and would hide ownership-mask bugs. + Rows stay unique within a token, which the union path relies on.""" + perm = torch.randperm(S, device="cuda", generator=g) + pool, spare = perm[:topk], perm[topk:] + n_keep = topk * 3 // 4 + n_fresh = topk - n_keep + assert spare.numel() >= n_fresh, "S must exceed topk enough to vary the set" + idx = torch.empty(T, topk, dtype=torch.int32, device="cuda") + for t in range(T): + keep = pool[torch.randperm(topk, device="cuda", generator=g)[:n_keep]] + fresh = spare[ + torch.randperm(spare.numel(), device="cuda", generator=g)[:n_fresh] + ] + idx[t] = torch.cat([keep, fresh]).to(torch.int32) + return idx + + +def _assert_matches(case, out, ref, tag, cos_min=0.999, max_abs=0.05): + cos = torch.nn.functional.cosine_similarity( + out.float().flatten(), ref.float().flatten(), dim=0 + ).item() + mabs = (out.float() - ref.float()).abs().max().item() + case.assertGreater(cos, cos_min, f"{tag}: cosine {cos:.6f}") + case.assertLess(mabs, max_abs, f"{tag}: max_abs {mabs:.4f}") + + +@unittest.skipIf(not torch.cuda.is_available(), "Test requires CUDA") +class TestDSATritonSparseMLAPrefill(CustomTestCase): + def test_base_path(self): + for T, topk in ((512, 512), (2048, 2048), (37, 2048)): + S = max(T, topk + 8) + q, kv, g = _qkv(T, S, 8, seed=T) + idx = _random_indices(T, topk, S, g) + _assert_matches( + self, + sparse_mla_prefill(q, kv, idx, SM_SCALE, D_V), + _reference(q, kv, idx), + f"base T={T} topk={topk}", + ) + + def test_ragged_minus_one_padding(self): + T, topk, S = 1024, 2048, 2056 + q, kv, g = _qkv(T, S, 8, seed=7) + idx = _random_indices(T, topk, S, g, pad_frac=0.6) + _assert_matches( + self, + sparse_mla_prefill(q, kv, idx, SM_SCALE, D_V), + _reference(q, kv, idx), + "ragged -1 padding", + ) + + def test_union_is_exact_on_overlapping_selections(self): + # The union tile is only exercised when neighbouring tokens share rows; + # each token must still see exactly its own set through the mask. T=1023 + # at G=4 leaves a 3-row tail that the per-token kernel must write into + # out[T_main:]; skipped or misplaced tail rows would come back as + # uninitialised memory or another token's output. + for group, T in ((2, 2048), (4, 2048), (4, 1023)): + topk, S = 2048, 4096 + q, kv, g = _qkv(T, S, 8, seed=100 + group) + idx = _overlapping_indices(T, topk, S, g) + _assert_matches( + self, + sparse_mla_prefill(q, kv, idx, SM_SCALE, D_V, union=group), + _reference(q, kv, idx), + f"union G={group} T={T}", + ) + + def test_large_head_count_steps_down_instead_of_oom(self): + # An oversized tile makes the first candidate fail on every GPU (on the + # CI GPU's 228 KB the tuned tile simply fits and nothing would step + # down), and the cache is cleared so an earlier test's fit cannot be + # moved to the front. The recorded fit must not be the requested tile. + _FIT_TILE.clear() + T, topk, S = 256, 512, 520 + q, kv, g = _qkv(T, S, 32, seed=9) + idx = _random_indices(T, topk, S, g) + _assert_matches( + self, + sparse_mla_prefill(q, kv, idx, SM_SCALE, D_V, config=(256, 8, 4)), + _reference(q, kv, idx), + "h=32 smem fallback", + ) + self.assertNotEqual(_FIT_TILE[("base", 32, 0, q.device.index)], (256, 4)) + + def test_union_large_head_count_steps_down(self): + # Same for the union launcher, whose Q tile is num_heads * G rows and + # runs out of shared memory sooner (16 heads at G=2 overflows SM120's + # 100 KB with the tuned tile). The oversized tile forces the step-down + # on every GPU; the union tile that fit must not be the requested one. + _FIT_TILE.clear() + T, topk, S = 512, 512, 1024 + q, kv, g = _qkv(T, S, 16, seed=21) + idx = _overlapping_indices(T, topk, S, g) + _assert_matches( + self, + sparse_mla_prefill( + q, kv, idx, SM_SCALE, D_V, union=2, union_config=(256, 4, 4) + ), + _reference(q, kv, idx), + "union G=2 h=16 smem fallback", + ) + self.assertNotEqual(_FIT_TILE[("union", 16, 2, q.device.index)], (256, 4)) + + def test_union_dedup_contract(self): + # Each group's rows must come out exactly once, with ownership bits equal + # to the OR of the tokens that selected them; -1 slots must be dropped, + # not counted. Checked against a Python set reference so a membership + # slip cannot hide behind the kernel's cosine gate. + G, K, S = 4, 64, 256 + g = torch.Generator(device="cuda").manual_seed(77) + idx = _overlapping_indices(3 * G, K, S, g) + idx[1, 40:] = -1 # a ragged row in the first group + idx[9, :] = -1 # a token with no selection at all + uidx, ubits, ulen = _union_dedup(idx, G) + self.assertEqual(tuple(uidx.shape), (3, G * K)) + for grp in range(3): + rows = {} + for tok in range(G): + for v in idx[grp * G + tok].tolist(): + if v >= 0: + rows[v] = rows.get(v, 0) | (1 << tok) + n = int(ulen[grp]) + self.assertEqual(n, len(rows), f"group {grp}: unique count") + got = dict(zip(uidx[grp, :n].tolist(), ubits[grp, :n].tolist())) + self.assertEqual(len(got), n, f"group {grp}: a row was emitted twice") + self.assertEqual(got, rows, f"group {grp}: rows or bits") + + def test_union_falls_back_when_tile_is_illegal(self): + # Regression: h=4 with G=2 is an 8-row tile, which tl.dot cannot run; + # the launcher only catches OutOfResources, so the compile error used to + # reach the caller. The kernel entry must take the per-token path instead + # and still match the reference. + T, topk, S = 256, 256, 512 + q, kv, g = _qkv(T, S, 4, seed=91) + idx = _overlapping_indices(T, topk, S, g) + _assert_matches( + self, + sparse_mla_prefill(q, kv, idx, SM_SCALE, D_V, union=2), + _reference(q, kv, idx), + "union G=2 h=4 falls back", + ) + + def test_int64_indexing_matches_int32(self): + # A KV pool past ~3.7M rows overflows int32 element offsets and the + # launcher switches to int64 gather addressing. No test can allocate + # that pool, so the mode is forced here instead: the two must agree + # bitwise, or a large deployment silently reads the wrong rows. + T, topk, S = 512, 512, 1024 + q, kv, g = _qkv(T, S, 8, seed=61) + idx = _random_indices(T, topk, S, g) + a = sparse_mla_prefill(q, kv, idx, SM_SCALE, D_V, int64_indexing=False) + b = sparse_mla_prefill(q, kv, idx, SM_SCALE, D_V, int64_indexing=True) + self.assertTrue(torch.equal(a, b), "int64 addressing changed the result") + _assert_matches(self, b, _reference(q, kv, idx), "int64 indexing") + + def test_deterministic(self): + # No split-K / atomics / partial merge, so repeated calls must be + # bitwise identical -- the property that lets a served model be + # reproduced run to run. + T, topk, S = 1024, 2048, 2056 + q, kv, g = _qkv(T, S, 8, seed=11) + idx = _random_indices(T, topk, S, g) + a = sparse_mla_prefill(q, kv, idx, SM_SCALE, D_V) + b = sparse_mla_prefill(q, kv, idx, SM_SCALE, D_V) + self.assertTrue(torch.equal(a, b), "kernel output is not deterministic") + + +@unittest.skipIf(not torch.cuda.is_available(), "Test requires CUDA") +class TestDSATritonPrefillBackendAdapter(CustomTestCase): + """The backend method driving the real kernel: values against the fp32 + reference and the `[num_tokens, num_heads, v_head_dim]` output contract + that `_forward_flashmla_sparse` returns to the same caller.""" + + def _backend(self, *, union=0): + from sglang.srt.layers.attention.dsa_backend import DeepseekSparseAttnBackend + + backend = DeepseekSparseAttnBackend.__new__(DeepseekSparseAttnBackend) + backend.dsa_triton_union = union + return backend + + def test_forward_matches_reference_and_output_contract(self): + T, topk, S, h = 1024, 512, 2048, 8 + q, kv, g = _qkv(T, S, h, seed=51) + idx = _overlapping_indices(T, topk, S, g) + ref = _reference(q, kv, idx) + + for union in (0, 2, 4): + with self.subTest(union=union): + out = self._backend(union=union)._forward_triton_sparse_mla( + q_all=q, + kv_cache=kv, + page_table_1=idx, + sm_scale=SM_SCALE, + v_head_dim=D_V, + ) + self.assertEqual( + tuple(out.shape), + (T, h, D_V), + "must match the [num_tokens, num_heads, v_head_dim] that " + "_forward_flashmla_sparse returns to the same caller", + ) + self.assertEqual(out.dtype, torch.bfloat16) + _assert_matches(self, out, ref, f"backend union={union}") + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/attention/test_triton_sparse_mla_prefill.py b/test/registered/unit/attention/test_triton_sparse_mla_prefill.py new file mode 100644 index 0000000000..210a05ba53 --- /dev/null +++ b/test/registered/unit/attention/test_triton_sparse_mla_prefill.py @@ -0,0 +1,322 @@ +# SPDX-License-Identifier: Apache-2.0 +"""CPU unit tests for the Triton sparse-MLA prefill adapter and its capability +check. The kernel is mocked, so these guard the wiring rather than the numerics +(which live in +``test/registered/kernels/ops/attention/test_dsa_triton_sparse_mla_prefill.py``): + +- argument marshalling between the DSA backend and the kernel entry point, +- the union switch being off unless asked for, at both the CLI layer and the + backend layer, forced off while the stream is capturing or under deterministic + inference, and left on inside a breakable-graph replay, +- the validator's accept/reject boundaries, +- that registering this backend does not change which backend SM120 selects on + its own. +""" + +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +import torch + +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=1, suite="base-a-test-cpu") + + +class TestTritonSparseMLAValidator(CustomTestCase): + """Boundaries of ``_validate_triton_sparse_mla_backend``.""" + + def _validate(self, **kwargs): + from sglang.srt.layers.attention.dsa_backend import ( + _validate_triton_sparse_mla_backend, + ) + + defaults = dict(device_sm_major=12, num_q_heads=8, union=0, index_kpool=1) + defaults.update(kwargs) + return _validate_triton_sparse_mla_backend(**defaults) + + def test_sm_major_boundary_is_hopper(self): + # SM80 and below are refused at startup. SM90+ is admitted: SM90 and + # SM120 run tuned tiles, other architectures the untuned default with a + # startup warning (see _PINNED in the kernel module). + with self.assertRaisesRegex(ValueError, "SM90"): + self._validate(device_sm_major=8) + for sm in (9, 10, 12): + self.assertIsNone(self._validate(device_sm_major=sm)) + + def test_union_group_size_contract(self): + for group in (0, 2, 4): + self.assertIsNone(self._validate(union=group)) + with self.assertRaisesRegex(ValueError, "must be 0, 2 or 4"): + self._validate(union=3) + + def test_union_tile_capacity(self): + # The union tile is at most 32 rows of num_q_heads * union; above that + # the kernel would fall back to the per-token path on every call, so + # it is a startup error instead of a silent no-op. + with self.assertRaisesRegex(ValueError, "union"): + self._validate(num_q_heads=16, union=4) + self.assertIsNone(self._validate(num_q_heads=16, union=2)) + + def test_union_tile_must_be_a_power_of_two_of_at_least_16(self): + # Regression: h=4 with union 2 (GLM-5 at TP16) and h=12 with union 2 + # passed the old `<= 32` check, then failed to compile on the first long + # prefill: `tl.arange` needs a power of two and `tl.dot` needs M >= 16. + # The launcher only catches OutOfResources, so the compile error reached + # the request. Both shapes must be refused at startup instead. + for heads, group in ((4, 2), (12, 2), (6, 4)): + with ( + self.subTest(heads=heads, group=group), + self.assertRaisesRegex(ValueError, "power of two"), + ): + self._validate(num_q_heads=heads, union=group) + for heads, group in ((4, 4), (8, 2), (8, 4), (16, 2)): + with self.subTest(heads=heads, group=group): + self.assertIsNone(self._validate(num_q_heads=heads, union=group)) + + def test_head_count_is_capped_at_measured_range(self): + # BLOCK_H = next_pow2(h) is never reduced by the smem step-down, so at + # h=64 / 128 the [BLOCK_H, 512] fp32 accumulator spills or misses the + # SM120 budget. Refuse what has not been measured rather than promise it. + self.assertIsNone(self._validate(num_q_heads=32)) + with self.assertRaisesRegex(ValueError, "32 query heads"): + self._validate(num_q_heads=33) + with self.assertRaisesRegex(ValueError, "32 query heads"): + self._validate(num_q_heads=128) + + def test_index_kpool_is_rejected_at_construction(self): + # Regression: only `flashmla_sparse` is rerouted for the pooled indexer + # tail, so this backend reached `_check_kpool_tail_backend` and raised + # NotImplementedError on the first prefill. Reject it at startup. + with self.assertRaisesRegex(ValueError, "index_kpool"): + self._validate(index_kpool=2) + self.assertIsNone(self._validate(index_kpool=1)) + + +class TestTritonSparseMLAUnionResolution(CustomTestCase): + """``_resolve_dsa_triton_union``: deterministic inference forces union off. + + Union makes a token's output depend on which tokens share its group and on + the ``T % G`` tail, which breaks the batch invariance deterministic inference + promises. The requested value must survive unchanged otherwise. + """ + + def _resolve(self, **kwargs): + from sglang.srt.layers.attention.dsa_backend import _resolve_dsa_triton_union + + return _resolve_dsa_triton_union(**kwargs) + + def test_deterministic_forces_union_off(self): + with self.assertLogs("sglang.srt.layers.attention.dsa_backend", "WARNING"): + self.assertEqual(self._resolve(union=4, deterministic=True), 0) + + def test_requested_value_kept_otherwise(self): + for union in (0, 2, 4): + self.assertEqual(self._resolve(union=union, deterministic=False), union) + self.assertEqual(self._resolve(union=0, deterministic=True), 0) + + +class TestTritonSparseMLAAdapter(CustomTestCase): + """The backend method forwards exactly what the kernel expects.""" + + def _call_forward( + self, *, union=0, capturing=False, stream_capturing=False, topk_length=None + ): + from sglang.srt.layers.attention.dsa_backend import DeepseekSparseAttnBackend + + captured = {} + + def _fake_kernel(q, kv, indices, sm_scale, d_v=512, **kwargs): + captured.update( + q=q, kv=kv, indices=indices, sm_scale=sm_scale, d_v=d_v, **kwargs + ) + return torch.zeros(q.shape[0], q.shape[1], d_v, dtype=torch.bfloat16) + + backend = DeepseekSparseAttnBackend.__new__(DeepseekSparseAttnBackend) + backend.dsa_triton_union = union + + # `get_is_capture_mode` is patched although the method no longer reads + # it, so the breakable-graph regression below stays a real check. + with ( + patch( + "sglang.kernels.ops.attention.dsa.triton_sparse_mla_prefill.sparse_mla_prefill", + _fake_kernel, + ), + patch( + "sglang.srt.model_executor.runner_utils.capture_mode.get_is_capture_mode", + lambda: capturing, + ), + patch("torch.cuda.is_current_stream_capturing", lambda: stream_capturing), + ): + out = backend._forward_triton_sparse_mla( + q_all=torch.zeros(4, 8, 576, dtype=torch.bfloat16), + kv_cache=torch.zeros(64, 576, dtype=torch.bfloat16), + page_table_1=torch.zeros(4, 16, dtype=torch.int32), + sm_scale=0.0625, + v_head_dim=512, + topk_length=topk_length, + ) + return captured, out + + def test_forwards_tensors_and_scale(self): + captured, out = self._call_forward() + self.assertEqual(tuple(captured["q"].shape), (4, 8, 576)) + self.assertEqual(tuple(captured["kv"].shape), (64, 576)) + self.assertEqual(tuple(captured["indices"].shape), (4, 16)) + self.assertEqual(captured["sm_scale"], 0.0625) + self.assertEqual(captured["d_v"], 512) + self.assertEqual(tuple(out.shape), (4, 8, 512)) + + def test_union_off_unless_requested(self): + captured, _ = self._call_forward() + self.assertEqual(captured["union"], 0) + + def test_union_is_plumbed_through(self): + captured, _ = self._call_forward(union=4) + self.assertEqual(captured["union"], 4) + + def test_union_stays_on_when_only_the_breakable_graph_flag_is_set(self): + # Regression: `get_is_capture_mode()` is true for the whole breakable + # prefill graph replay, during which attention runs eagerly between the + # graph segments. Gating on it switched union off for every replayed + # prefill batch, i.e. for every bucket the default prefill graph covers. + captured, _ = self._call_forward(union=4, capturing=True) + self.assertEqual(captured["union"], 4) + + def test_union_is_disabled_when_the_stream_itself_is_capturing(self): + # Under `--cuda-graph-backend-prefill full` the attention runs inside + # stream capture; the union path has not been validated there, so the + # per-token path (same result) must run instead. + captured, _ = self._call_forward( + union=4, capturing=False, stream_capturing=True + ) + self.assertEqual(captured["union"], 0) + + def test_topk_length_is_forwarded_when_rows_match(self): + # Without it the kernel rebuilds the per-row valid count on every layer + # (~64 MB of temporaries at T=8192, topk=2048) from data the backend + # already holds in `dsa_cache_seqlens_int32`. + lengths = torch.full((4,), 16, dtype=torch.int32) + captured, _ = self._call_forward(topk_length=lengths) + self.assertIs(captured["topk_length"], lengths) + + def test_topk_length_is_dropped_when_rows_diverge(self): + # Same contract as `_forward_flashmla_sparse`: metadata rows that do not + # match q rows fall back to full-width compute instead of misindexing. + captured, _ = self._call_forward( + topk_length=torch.full((5,), 16, dtype=torch.int32) + ) + self.assertIsNone(captured["topk_length"]) + + +class TestTritonSparseMLATopkTransformRouting(CustomTestCase): + """The prefill top-k transform must be RAGGED for this backend. + + Regression: the backend dequantizes the KV cache inside the RAGGED branch of + `forward_extend`, and its kernel is bf16-only. When `get_topk_transform_method` + left it on PAGED, the branch was skipped and the raw packed FP8 pool reached + the kernel, which died on `Unsupported rhs dtype fp8e4nv` mid-forward. Only an + end-to-end run caught it: the dispatch branch reads correct in isolation. + """ + + def _method(self, prefill_impl, *, store_fp8=True, mode=None): + from sglang.srt.layers.attention.dsa_backend import DeepseekSparseAttnBackend + from sglang.srt.model_executor.forward_batch_info import ForwardMode + + backend = DeepseekSparseAttnBackend.__new__(DeepseekSparseAttnBackend) + backend.dsa_kv_cache_store_fp8 = store_fp8 + backend.dsa_prefill_impl = prefill_impl + return backend.get_topk_transform_method( + ForwardMode.EXTEND if mode is None else mode + ) + + def test_extend_uses_ragged_for_bf16_kv_backends(self): + from sglang.srt.layers.attention.dsa_backend import TopkTransformMethod + + for impl in ("flashmla_sparse", "flashmla_sparse_q8", "triton_sparse_mla"): + with self.subTest(impl=impl): + self.assertEqual(self._method(impl), TopkTransformMethod.RAGGED, impl) + + def test_other_backends_keep_paged(self): + from sglang.srt.layers.attention.dsa_backend import TopkTransformMethod + + self.assertEqual( + self._method("flashinfer_sparse_mla"), TopkTransformMethod.PAGED + ) + + +class TestTritonSparseMLARegistration(CustomTestCase): + """Selectable from the CLI, and opt-in only.""" + + def test_choice_is_registered(self): + import argparse + + from sglang.srt.server_args import ServerArgs + + parser = argparse.ArgumentParser() + ServerArgs.add_cli_args(parser) + actions_by_option = { + option: action + for action in parser._actions + for option in action.option_strings + } + self.assertIn( + "triton_sparse_mla", actions_by_option["--dsa-prefill-backend"].choices + ) + + def test_union_defaults_to_off(self): + from sglang.srt.server_args import ServerArgs + + self.assertEqual(ServerArgs(model_path="dummy").dsa_triton_union, 0) + + def test_sm120_glm_fp8_still_resolves_to_flashinfer_on_its_own(self): + # Registering this backend must not change what SM120 selects by itself: + # the GLM FP8-KV arm of the resolver declares flashinfer_sparse_mla for + # both phases unless the user set one, and a user-set prefill backend + # is kept as given. + from sglang.srt.arg_groups import overrides + + hf_config = SimpleNamespace( + architectures=["GlmMoeDsaForCausalLM"], learnable_sink=False + ) + platform = SimpleNamespace(is_npu=False, is_xpu=False, is_hip=False) + + def resolve(prefill): + view = SimpleNamespace( + dsa_prefill_backend=prefill, + dsa_decode_backend=None, + kv_cache_dtype="fp8_e4m3", + enable_hisparse=False, + ) + with ( + patch.object( + overrides, + "model_config_of", + return_value=SimpleNamespace(hf_config=hf_config), + ), + patch.object(overrides, "get_platform", return_value=platform), + patch( + "sglang.srt.configs.model_config.is_deepseek_dsa", + return_value=True, + ), + patch("torch.cuda.get_device_capability", return_value=(12, 0)), + ): + return overrides._dsa_split_backend_resolution(view) + + self.assertEqual( + resolve(None), + { + "dsa_prefill_backend": "flashinfer_sparse_mla", + "dsa_decode_backend": "flashinfer_sparse_mla", + }, + ) + self.assertEqual( + resolve("triton_sparse_mla"), + {"dsa_decode_backend": "flashinfer_sparse_mla"}, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/test_flashinfer_sparse_mla.py b/test/registered/unit/test_flashinfer_sparse_mla.py index 1b72d94403..2f770adf16 100644 --- a/test/registered/unit/test_flashinfer_sparse_mla.py +++ b/test/registered/unit/test_flashinfer_sparse_mla.py @@ -105,6 +105,14 @@ class TestFlashInferSparseMLABackendGate(unittest.TestCase): with self.assertRaisesRegex(ValueError, "only flashinfer_sparse_mla"): self._validate(prefill, decode) + def test_accepts_triton_sparse_mla_for_prefill(self): + # triton_sparse_mla is selectable for prefill on this platform while + # flashinfer stays the decode kernel; the allowlist widened for exactly + # that pair and no other, so a third backend must still be refused. + self.assertTrue(self._validate("triton_sparse_mla", "flashinfer_sparse_mla")) + with self.assertRaisesRegex(ValueError, "only flashinfer_sparse_mla"): + self._validate("triton_sparse_mla", "trtllm") + def test_reports_unsupported_configuration(self): with self.assertRaises(ValueError) as error: self._validate(