[SM120&90] Add CUDA fused Triton sparse-MLA prefill backend for DSA (#32779)
Co-authored-by: Claude Fable 5 <noreply@anthropic.com> Co-authored-by: Po-Han Huang (NVIDIA) <53919306+nvpohanh@users.noreply.github.com> Co-authored-by: Po-Han Huang <pohanh@nvidia.com>
This commit is contained in:
co-authored by
Claude Fable 5
Po-Han Huang
Po-Han Huang
parent
1e1bdeedbb
commit
e446698cdd
@@ -0,0 +1,511 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the SGLang project
|
||||
"""Fused Triton sparse-MLA prefill for DeepSeek Sparse Attention (DSA) on CUDA.
|
||||
|
||||
One Triton program per query token: the heads are padded into a 16-row MMA tile
|
||||
rather than into the grid, the online softmax stays in registers, and ``V`` is
|
||||
read as ``kv[:, :d_v]``, so there are no partials and no merge pass.
|
||||
``dsa/triton_sparse_mla.py`` is the gfx950-only FP8 sibling of this bf16 module.
|
||||
|
||||
Interface matches ``sgl_kernel.flash_mla.flash_mla_sparse_fwd``::
|
||||
|
||||
q [T, H, 576] bf16 (absorbed MLA: 512 nope + 64 rope)
|
||||
kv [S, 576] bf16 (V is kv[:, :512])
|
||||
indices [T, topk] int32 (-1 marks an invalid slot; no upper-bound check)
|
||||
out [T, H, 512] bf16
|
||||
|
||||
A row of ``indices`` must not name the same KV position twice. Top-k selection
|
||||
cannot, so this holds for every DSA caller; it is stated because ``union``
|
||||
gathers the distinct union of G rows and weights each position once, whereas
|
||||
the base path would weight a repeat twice.
|
||||
|
||||
``union`` (opt-in, 2 or 4): G adjacent query tokens share one gathered union
|
||||
index set, and a per-row ownership bitmask restores each token's own softmax
|
||||
support, so the result is the per-token result up to accumulation order. It
|
||||
pays off because neighbouring tokens' selections overlap heavily on real indexer
|
||||
output; the measurements are in the PR that added this module.
|
||||
|
||||
All tunables are explicit arguments; the module reads no environment variables.
|
||||
If a tuned tile exceeds the device shared-memory budget, the launcher steps down
|
||||
through smaller tiles instead of raising ``OutOfResources``.
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Swept on the hardware named; anything else gets _UNTUNED_DEFAULT and a warning.
|
||||
_PINNED = {
|
||||
(9, 0): (64, 8, 2), # SM90, swept at T=8192
|
||||
(12, 0): (64, 4, 2), # SM120, swept at T=8192
|
||||
(12, 1): (64, 4, 2), # SM121: SM120's tile, not swept separately
|
||||
}
|
||||
_UNTUNED_DEFAULT = (64, 8, 3)
|
||||
_UNTUNED_ARCH_WARNED = set()
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _sparse_mla_prefill_kernel(
|
||||
q_ptr,
|
||||
kv_ptr,
|
||||
idx_ptr,
|
||||
len_ptr,
|
||||
o_ptr,
|
||||
sm_scale,
|
||||
topk,
|
||||
H: tl.constexpr,
|
||||
BLOCK_H: tl.constexpr,
|
||||
D_QK: tl.constexpr,
|
||||
D_V: tl.constexpr,
|
||||
BLOCK_N: tl.constexpr,
|
||||
IDX64: tl.constexpr,
|
||||
):
|
||||
t = tl.program_id(0)
|
||||
# Token offsets in int64: at h=128 the q offset t*H*D_QK passes 2^31 from
|
||||
# t=29128 and the output offset from t=32768, wrapping to another token.
|
||||
t64 = t.to(tl.int64)
|
||||
D_TAIL: tl.constexpr = D_QK - D_V
|
||||
|
||||
h = tl.arange(0, BLOCK_H)
|
||||
hmask = h < H
|
||||
dv = tl.arange(0, D_V)
|
||||
dt = tl.arange(0, D_TAIL)
|
||||
|
||||
qb = q_ptr + t64 * H * D_QK
|
||||
q_main = tl.load(
|
||||
qb + h[:, None] * D_QK + dv[None, :], mask=hmask[:, None], other=0.0
|
||||
)
|
||||
q_tail = tl.load(
|
||||
qb + h[:, None] * D_QK + (D_V + dt)[None, :], mask=hmask[:, None], other=0.0
|
||||
)
|
||||
|
||||
m_i = tl.full([BLOCK_H], -float("inf"), tl.float32)
|
||||
l_i = tl.zeros([BLOCK_H], tl.float32)
|
||||
acc = tl.zeros([BLOCK_H, D_V], tl.float32)
|
||||
|
||||
n = tl.arange(0, BLOCK_N)
|
||||
k_len = tl.load(len_ptr + t)
|
||||
for k0 in tl.range(0, k_len, BLOCK_N):
|
||||
idx = tl.load(idx_ptr + t64 * topk + k0 + n, mask=(k0 + n) < k_len, other=-1)
|
||||
valid = idx >= 0
|
||||
if IDX64:
|
||||
row = tl.where(valid, idx, 0).to(tl.int64)
|
||||
else:
|
||||
row = tl.where(valid, idx, 0)
|
||||
kb = kv_ptr + row[:, None] * D_QK
|
||||
kv_main = tl.load(kb + dv[None, :], mask=valid[:, None], other=0.0)
|
||||
kv_tail = tl.load(kb + (D_V + dt)[None, :], mask=valid[:, None], other=0.0)
|
||||
|
||||
qk = tl.dot(q_main, tl.trans(kv_main))
|
||||
qk = tl.dot(q_tail, tl.trans(kv_tail), qk) * sm_scale
|
||||
qk = tl.where(valid[None, :], qk, -float("inf"))
|
||||
|
||||
m_new = tl.maximum(m_i, tl.max(qk, axis=1))
|
||||
m_safe = tl.where(m_new == -float("inf"), 0.0, m_new)
|
||||
alpha = tl.exp(m_i - m_safe)
|
||||
p = tl.exp(qk - m_safe[:, None])
|
||||
l_i = l_i * alpha + tl.sum(p, axis=1)
|
||||
acc = acc * alpha[:, None] + tl.dot(p.to(kv_main.dtype), kv_main)
|
||||
m_i = m_new
|
||||
|
||||
l_safe = tl.where(l_i == 0.0, 1.0, l_i)
|
||||
acc = acc * (1.0 / l_safe[:, None])
|
||||
tl.store(
|
||||
o_ptr + t64 * H * D_V + h[:, None] * D_V + dv[None, :],
|
||||
acc.to(o_ptr.dtype.element_ty),
|
||||
mask=hmask[:, None],
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Union path: G adjacent tokens share one gathered KV set. Per group, the G*K
|
||||
# selected rows are sorted and deduplicated on the device, and an ownership bit
|
||||
# per unique row lets the kernel below restore each token's own softmax
|
||||
# support. The cost is O(G*K*log K) per group whatever the KV span, so
|
||||
# nothing here depends on the index range, reads back to the host, or
|
||||
# persists across calls.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _union_dedup_kernel(
|
||||
sorted_ptr,
|
||||
uidx_ptr,
|
||||
ubits_ptr,
|
||||
ulen_ptr,
|
||||
K: tl.constexpr,
|
||||
G: tl.constexpr,
|
||||
LOG_K: tl.constexpr,
|
||||
):
|
||||
# One program per group of G tokens whose K selected rows are each sorted
|
||||
# ascending (-1 pads sort first). Token i emits row v if no earlier token
|
||||
# selected it; its ownership bits OR in every token that did. Membership is
|
||||
# a vectorised lower_bound over the other token's list, which stays in L1.
|
||||
g = tl.program_id(0).to(tl.int64)
|
||||
grp = sorted_ptr + g * G * K
|
||||
n = tl.arange(0, K)
|
||||
cursor = tl.zeros([], tl.int32)
|
||||
for i in tl.static_range(G):
|
||||
v = tl.load(grp + i * K + n)
|
||||
valid = v >= 0
|
||||
bits = tl.where(valid, 1 << i, 0).to(tl.int32)
|
||||
first = valid
|
||||
for j in tl.static_range(G):
|
||||
if j != i:
|
||||
lst = grp + j * K
|
||||
lo = tl.zeros([K], tl.int32)
|
||||
hi = tl.full([K], K, tl.int32)
|
||||
for _ in tl.static_range(LOG_K):
|
||||
mid = (lo + hi) // 2
|
||||
m = tl.load(lst + mid, mask=mid < K, other=2147483647)
|
||||
lt = m < v
|
||||
lo = tl.where(lt, mid + 1, lo)
|
||||
hi = tl.where(lt, hi, mid)
|
||||
found = (tl.load(lst + lo, mask=lo < K, other=-2) == v) & valid
|
||||
bits |= tl.where(found, 1 << j, 0).to(tl.int32)
|
||||
if j < i:
|
||||
first = first & (found == 0)
|
||||
wpos = cursor + tl.cumsum(first.to(tl.int32), axis=0) - first.to(tl.int32)
|
||||
tl.store(uidx_ptr + g * (G * K) + wpos, v, mask=first)
|
||||
tl.store(ubits_ptr + g * (G * K) + wpos, bits, mask=first)
|
||||
cursor += tl.sum(first.to(tl.int32))
|
||||
tl.store(ulen_ptr + g, cursor)
|
||||
|
||||
|
||||
# Per-group unique rows and ownership bits for [NG*G, K] indices: uidx [NG, G*K]
|
||||
# int32 holds each group's distinct rows in its first ulen[g] slots (token-major,
|
||||
# each token's new rows ascending, the rest unspecified), ubits [NG, G*K] int32
|
||||
# has bit i set when token i selected the row, ulen [NG] int32. torch sorts each
|
||||
# K-wide row in shared memory; one Triton pass does the cross-token membership.
|
||||
def _union_dedup(idx_main, G):
|
||||
T_main, K = idx_main.shape
|
||||
NG = T_main // G
|
||||
srt = torch.sort(idx_main, dim=1)[0]
|
||||
uidx = torch.empty(NG, G * K, dtype=torch.int32, device=idx_main.device)
|
||||
ubits = torch.empty_like(uidx)
|
||||
ulen = torch.empty(NG, dtype=torch.int32, device=idx_main.device)
|
||||
_union_dedup_kernel[(NG,)](
|
||||
srt,
|
||||
uidx,
|
||||
ubits,
|
||||
ulen,
|
||||
K=K,
|
||||
G=G,
|
||||
# log2(K) + 1 halvings: the last one resolves a one-element interval.
|
||||
LOG_K=K.bit_length(),
|
||||
num_warps=16 if G == 4 else 8, # swept on SM90 at K=2048
|
||||
)
|
||||
return uidx, ubits, ulen
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _sparse_mla_prefill_union_kernel(
|
||||
q_ptr,
|
||||
kv_ptr,
|
||||
uidx_ptr,
|
||||
ubits_ptr,
|
||||
ulen_ptr,
|
||||
o_ptr,
|
||||
sm_scale,
|
||||
U_STRIDE,
|
||||
H: tl.constexpr,
|
||||
G: tl.constexpr,
|
||||
D_QK: tl.constexpr,
|
||||
D_V: tl.constexpr,
|
||||
BLOCK_N: tl.constexpr,
|
||||
):
|
||||
g = tl.program_id(0)
|
||||
D_TAIL: tl.constexpr = D_QK - D_V
|
||||
GH: tl.constexpr = G * H
|
||||
|
||||
r = tl.arange(0, GH)
|
||||
tok_of_row = r // H
|
||||
dv = tl.arange(0, D_V)
|
||||
dt = tl.arange(0, D_TAIL)
|
||||
|
||||
qb = q_ptr + g.to(tl.int64) * GH * D_QK
|
||||
q_main = tl.load(qb + r[:, None] * D_QK + dv[None, :])
|
||||
q_tail = tl.load(qb + r[:, None] * D_QK + (D_V + dt)[None, :])
|
||||
|
||||
m_i = tl.full([GH], -float("inf"), tl.float32)
|
||||
l_i = tl.zeros([GH], tl.float32)
|
||||
acc = tl.zeros([GH, D_V], tl.float32)
|
||||
|
||||
n = tl.arange(0, BLOCK_N)
|
||||
u_len = tl.load(ulen_ptr + g)
|
||||
ub = g.to(tl.int64) * U_STRIDE
|
||||
for k0 in tl.range(0, u_len, BLOCK_N):
|
||||
inb = (k0 + n) < u_len
|
||||
uidx = tl.load(uidx_ptr + ub + k0 + n, mask=inb, other=-1)
|
||||
bits = tl.load(ubits_ptr + ub + k0 + n, mask=inb, other=0)
|
||||
valid = uidx >= 0
|
||||
row = tl.where(valid, uidx, 0).to(tl.int64)
|
||||
kb = kv_ptr + row * D_QK
|
||||
kv_main = tl.load(kb[:, None] + dv[None, :], mask=valid[:, None], other=0.0)
|
||||
kv_tail = tl.load(
|
||||
kb[:, None] + (D_V + dt)[None, :], mask=valid[:, None], other=0.0
|
||||
)
|
||||
|
||||
qk = tl.dot(q_main, tl.trans(kv_main))
|
||||
qk = tl.dot(q_tail, tl.trans(kv_tail), qk) * sm_scale
|
||||
sel = ((bits[None, :] >> tok_of_row[:, None]) & 1) != 0
|
||||
qk = tl.where(sel & valid[None, :], qk, -float("inf"))
|
||||
|
||||
m_new = tl.maximum(m_i, tl.max(qk, axis=1))
|
||||
m_safe = tl.where(m_new == -float("inf"), 0.0, m_new)
|
||||
alpha = tl.exp(m_i - m_safe)
|
||||
p = tl.exp(qk - m_safe[:, None])
|
||||
l_i = l_i * alpha + tl.sum(p, axis=1)
|
||||
acc = acc * alpha[:, None] + tl.dot(p.to(kv_main.dtype), kv_main)
|
||||
m_i = m_new
|
||||
|
||||
l_safe = tl.where(l_i == 0.0, 1.0, l_i)
|
||||
acc = acc / l_safe[:, None]
|
||||
tl.store(
|
||||
o_ptr + g.to(tl.int64) * GH * D_V + r[:, None] * D_V + dv[None, :],
|
||||
acc.to(o_ptr.dtype.element_ty),
|
||||
)
|
||||
|
||||
|
||||
# Returns True if handled; the T % G tail rows go through the per-token path.
|
||||
def _union_path(*, q, kv, indices, sm_scale, d_v, out, G, union_config):
|
||||
T, h, d_qk = q.shape
|
||||
K = indices.shape[-1]
|
||||
rows = G * h
|
||||
# tl.arange needs a power of two and tl.dot needs M >= 16; the tuned tiles
|
||||
# stop at 32 rows. The backend validator rejects the same shapes at startup.
|
||||
if G < 2 or rows < 16 or rows > 32 or rows & (rows - 1):
|
||||
return False
|
||||
if K & (K - 1): # the dedup kernel tiles each token's list with tl.arange(0, K)
|
||||
return False
|
||||
T_main = (T // G) * G
|
||||
if T_main == 0:
|
||||
return False
|
||||
NG = T_main // G
|
||||
uidx, ubits, ulen = _union_dedup(idx_main=indices[:T_main], G=G)
|
||||
U_STRIDE = G * K
|
||||
if union_config is not None:
|
||||
bn, warps, stages = union_config
|
||||
elif torch.cuda.get_device_capability(q.device)[0] >= 12:
|
||||
# SM120 sweeps on real indices: (64,4,3) at G=2. At G=4 the 32-row Q
|
||||
# tile makes BN=64 exceed the 100 KB budget; (32,4,2) beats every neighbour.
|
||||
bn, warps, stages = (64, 4, 3) if G == 2 else (32, 4, 2)
|
||||
else:
|
||||
bn, warps, stages = (64, 4, 2) if G == 4 else (64, 8, 2)
|
||||
# The union Q tile is G*H rows, so its shared-memory footprint grows with
|
||||
# the head count (16 heads at G=2 exceeds SM120's 100 KB with the tuned
|
||||
# tile). Step down like the per-token launcher rather than fail the request.
|
||||
for bn_try, ns_try in _tile_candidates("union", h, G, q.device, bn, stages):
|
||||
try:
|
||||
_sparse_mla_prefill_union_kernel[(NG,)](
|
||||
q[:T_main],
|
||||
kv,
|
||||
uidx,
|
||||
ubits,
|
||||
ulen,
|
||||
out[:T_main],
|
||||
sm_scale,
|
||||
U_STRIDE,
|
||||
H=h,
|
||||
G=G,
|
||||
D_QK=d_qk,
|
||||
D_V=d_v,
|
||||
BLOCK_N=bn_try,
|
||||
num_warps=warps,
|
||||
num_stages=ns_try,
|
||||
)
|
||||
except triton.runtime.errors.OutOfResources:
|
||||
continue
|
||||
_FIT_TILE[("union", h, G, q.device.index)] = (bn_try, ns_try)
|
||||
break
|
||||
else:
|
||||
return False # no tile fits; caller falls through to the per-token path
|
||||
if T_main < T:
|
||||
sparse_mla_prefill(
|
||||
q[T_main:],
|
||||
kv,
|
||||
indices[T_main:],
|
||||
sm_scale,
|
||||
d_v,
|
||||
out=out[T_main:],
|
||||
union=0,
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
def _topk_length(indices, topk):
|
||||
valid = indices >= 0
|
||||
any_valid = valid.any(dim=-1)
|
||||
last = topk - torch.flip(valid, [-1]).int().argmax(dim=-1)
|
||||
return torch.where(any_valid, last, torch.zeros_like(last)).to(torch.int32)
|
||||
|
||||
|
||||
# Per-arch tuned (BLOCK_N, num_warps, num_stages); see _PINNED.
|
||||
def _config(device):
|
||||
cap = torch.cuda.get_device_capability(device)
|
||||
if cap in _PINNED:
|
||||
return _PINNED[cap]
|
||||
if cap not in _UNTUNED_ARCH_WARNED:
|
||||
_UNTUNED_ARCH_WARNED.add(cap)
|
||||
# The kernel is correct on any SM90+ device, but the tile was only swept
|
||||
# on the architectures in _PINNED. Say so rather than quietly running a
|
||||
# config nobody has measured; add an entry there once swept.
|
||||
logger.warning(
|
||||
"triton_sparse_mla: no tuned tile for sm_%d%d; falling back to %s. "
|
||||
"Sweep and add it to _PINNED for best throughput.",
|
||||
cap[0],
|
||||
cap[1],
|
||||
_UNTUNED_DEFAULT,
|
||||
)
|
||||
return _UNTUNED_DEFAULT
|
||||
|
||||
|
||||
# Ordered (BLOCK_N, num_stages) candidates: the tuned config first, then
|
||||
# progressively smaller shared-memory footprints, so one pinned config serves
|
||||
# head counts and devices whose budget the tuned tile would exceed.
|
||||
def _smem_fallbacks(bn, stages):
|
||||
seen, out = set(), []
|
||||
for cand in (
|
||||
(bn, stages),
|
||||
(bn, 2),
|
||||
(bn // 2, stages),
|
||||
(bn // 2, 2),
|
||||
(bn // 4, 2),
|
||||
(16, 2),
|
||||
):
|
||||
b, ns = max(16, cand[0]), max(1, cand[1])
|
||||
if (b, ns) not in seen:
|
||||
seen.add((b, ns))
|
||||
out.append((b, ns))
|
||||
return out
|
||||
|
||||
|
||||
# The tile that fit last time, per (path, num_heads, group, device index). The
|
||||
# tuned tile can exceed a device's shared memory (h=32 on SM120's 100 KB); the
|
||||
# step-down must then run once per shape, not on every layer of every prefill.
|
||||
_FIT_TILE = {}
|
||||
|
||||
|
||||
# _smem_fallbacks with the tile that fit last time moved to the front.
|
||||
def _tile_candidates(path, h, G, device, bn, stages):
|
||||
cands = _smem_fallbacks(bn, stages)
|
||||
fit = _FIT_TILE.get((path, h, G, device.index))
|
||||
if fit in cands:
|
||||
cands.remove(fit)
|
||||
cands.insert(0, fit)
|
||||
return cands
|
||||
|
||||
|
||||
def sparse_mla_prefill(
|
||||
q,
|
||||
kv,
|
||||
indices,
|
||||
sm_scale,
|
||||
d_v=512,
|
||||
*,
|
||||
topk_length=None,
|
||||
out=None,
|
||||
union=0,
|
||||
config=None,
|
||||
union_config=None,
|
||||
int64_indexing=None,
|
||||
):
|
||||
"""Fused sparse-MLA prefill. Returns ``out`` ``[T, H, d_v]`` bf16.
|
||||
|
||||
Args:
|
||||
q: ``[T, H, d_qk]`` bf16 query (absorbed MLA; ``d_qk = d_v + rope``).
|
||||
kv: ``[S, d_qk]`` or ``[S, 1, d_qk]`` bf16 latent cache; ``V`` is
|
||||
``kv[:, :d_v]`` (no separate value gather).
|
||||
indices: ``[T, topk]`` or ``[T, 1, topk]`` int32 selected slots; ``-1``
|
||||
marks an invalid slot and is skipped. Every other value must be a
|
||||
row of ``kv``: the kernels make no upper-bound check.
|
||||
sm_scale: softmax scale.
|
||||
d_v: value head dim (512 for DSA).
|
||||
topk_length: optional ``[T]`` int32 per-row valid count. Computed from
|
||||
``indices`` when omitted; pass it to skip that reduction.
|
||||
out: optional preallocated ``[T, H, d_v]`` bf16 output.
|
||||
union: 0 (off), 2 or 4: share one gathered union index set across ``G``
|
||||
adjacent query tokens. Exact, not an approximation: an ownership
|
||||
bitmask restores each token's own softmax support.
|
||||
config / union_config: optional tile overrides
|
||||
``(BLOCK_N, num_warps, num_stages)``. Defaults are the per-arch
|
||||
tuned entries in ``_PINNED``.
|
||||
"""
|
||||
if kv.dim() == 3: # [S, 1, D] -> [S, D]
|
||||
assert kv.shape[1] == 1
|
||||
kv = kv.squeeze(1)
|
||||
if indices.dim() == 3: # [T, 1, K] -> [T, K]
|
||||
assert indices.shape[1] == 1
|
||||
indices = indices.squeeze(1)
|
||||
T, h, d_qk = q.shape
|
||||
topk = indices.shape[-1]
|
||||
q, kv, indices = q.contiguous(), kv.contiguous(), indices.contiguous()
|
||||
if kv.dtype != torch.bfloat16:
|
||||
# An FP8 pool that reaches this entry (the PAGED transform leaves it
|
||||
# packed) would otherwise fail in Triton compile or read fp8 bytes at a
|
||||
# 576-element stride.
|
||||
raise TypeError(
|
||||
f"sparse_mla_prefill needs a bf16 KV cache, got {kv.dtype}; "
|
||||
"dequantize the pool first."
|
||||
)
|
||||
if out is None:
|
||||
out = torch.empty(T, h, d_v, dtype=torch.bfloat16, device=q.device)
|
||||
|
||||
if union in (2, 4) and _union_path(
|
||||
q=q,
|
||||
kv=kv,
|
||||
indices=indices,
|
||||
sm_scale=sm_scale,
|
||||
d_v=d_v,
|
||||
out=out,
|
||||
G=union,
|
||||
union_config=union_config,
|
||||
):
|
||||
return out
|
||||
|
||||
if topk_length is None:
|
||||
topk_length = _topk_length(indices, topk)
|
||||
|
||||
bn, warps, stages = config or _config(q.device)
|
||||
# int32 gather addressing keeps the gather loop on IMAD; int64 only when the
|
||||
# pool could overflow int32 element offsets (row*d_qk + d_qk-1), i.e. above
|
||||
# ~3.7M rows at d_qk=576. No test can allocate that pool, so the mode is
|
||||
# overridable to keep the int64 path reachable from a test.
|
||||
if int64_indexing is None:
|
||||
idx64 = kv.shape[0] > (2**31 - 1 - (d_qk - 1)) // d_qk
|
||||
else:
|
||||
idx64 = bool(int64_indexing)
|
||||
block_h = max(16, triton.next_power_of_2(h))
|
||||
for bn_try, ns_try in _tile_candidates("base", h, 0, q.device, bn, stages):
|
||||
try:
|
||||
_sparse_mla_prefill_kernel[(T,)](
|
||||
q,
|
||||
kv,
|
||||
indices,
|
||||
topk_length,
|
||||
out,
|
||||
sm_scale,
|
||||
topk,
|
||||
H=h,
|
||||
BLOCK_H=block_h,
|
||||
D_QK=d_qk,
|
||||
D_V=d_v,
|
||||
BLOCK_N=bn_try,
|
||||
num_warps=warps,
|
||||
num_stages=ns_try,
|
||||
IDX64=idx64,
|
||||
)
|
||||
except triton.runtime.errors.OutOfResources:
|
||||
# A larger head count (BLOCK_H) or a smaller smem budget can push the
|
||||
# pinned tile over the device limit (e.g. h=32 on SM120's 100 KB).
|
||||
# Step down the K tile / pipeline depth instead of failing the request.
|
||||
continue
|
||||
_FIT_TILE[("base", h, 0, q.device.index)] = (bn_try, ns_try)
|
||||
return out
|
||||
raise triton.runtime.errors.OutOfResources(
|
||||
0, 0, "shared memory: no fallback config fits this device/shape"
|
||||
)
|
||||
@@ -700,12 +700,15 @@ def _validate_flashinfer_sparse_mla_backend(
|
||||
f"decode_impl={decode_impl!r}."
|
||||
)
|
||||
if is_glm_sm12_fp8:
|
||||
unsupported = selected - {"flashinfer_sparse_mla"}
|
||||
# flashinfer_sparse_mla stays the auto-selected default on this
|
||||
# platform; triton_sparse_mla is a validated alternative prefill
|
||||
# implementation that the user may select explicitly.
|
||||
unsupported = selected - {"flashinfer_sparse_mla", "triton_sparse_mla"}
|
||||
if unsupported:
|
||||
raise ValueError(
|
||||
"GLM DSA with FP8 KV cache on NVIDIA SM120/SM121 supports "
|
||||
"only flashinfer_sparse_mla, "
|
||||
f"but got {sorted(unsupported)}."
|
||||
"only flashinfer_sparse_mla (default) or triton_sparse_mla "
|
||||
f"(prefill), but got {sorted(unsupported)}."
|
||||
)
|
||||
return uses_flashinfer_sparse_mla
|
||||
|
||||
|
||||
@@ -217,6 +217,7 @@ class ExecKernel(msgspec.Struct):
|
||||
"flashmla_kv",
|
||||
"flashmla_auto",
|
||||
"flashinfer_sparse_mla",
|
||||
"triton_sparse_mla",
|
||||
"fa3",
|
||||
"tilelang",
|
||||
"triton",
|
||||
@@ -237,6 +238,16 @@ class ExecKernel(msgspec.Struct):
|
||||
choices=["auto", "flashmla_sparse", "flashmla_sparse_q8"],
|
||||
),
|
||||
] = "auto"
|
||||
dsa_triton_union: A[
|
||||
int,
|
||||
Arg(
|
||||
help="Union group size for --dsa-prefill-backend triton_sparse_mla: "
|
||||
"0 disables, 2 or 4 make G adjacent query tokens share one gathered "
|
||||
"index set (mathematically exact; an ownership mask restores each "
|
||||
"token's own softmax support).",
|
||||
choices=[0, 2, 4],
|
||||
),
|
||||
] = 0
|
||||
dsa_decode_backend: A[
|
||||
Optional[str],
|
||||
Arg(
|
||||
|
||||
@@ -330,6 +330,7 @@ _DSA_IMPL_T: TypeAlias = Literal[
|
||||
"flashmla_sparse_q8",
|
||||
"flashmla_kv",
|
||||
"flashinfer_sparse_mla",
|
||||
"triton_sparse_mla",
|
||||
"fa3",
|
||||
"tilelang",
|
||||
"triton",
|
||||
@@ -338,6 +339,71 @@ _DSA_IMPL_T: TypeAlias = Literal[
|
||||
]
|
||||
|
||||
|
||||
# Largest head count the kernel has been measured at. Above it BLOCK_H=64/128
|
||||
# makes the [BLOCK_H, 512] fp32 accumulator spill or miss SM120's smem budget.
|
||||
_TRITON_SPARSE_MLA_MAX_HEADS = 32
|
||||
|
||||
|
||||
def _validate_triton_sparse_mla_backend(
|
||||
*,
|
||||
device_sm_major: int,
|
||||
num_q_heads: int,
|
||||
union: int,
|
||||
index_kpool: int,
|
||||
) -> None:
|
||||
if _is_hip:
|
||||
raise ValueError(
|
||||
"triton_sparse_mla is a CUDA prefill backend; on ROCm use the "
|
||||
"existing tilelang / aiter DSA paths."
|
||||
)
|
||||
if device_sm_major < 9:
|
||||
raise ValueError(
|
||||
"triton_sparse_mla requires SM90 (Hopper) or newer; "
|
||||
f"got sm_major={device_sm_major}."
|
||||
)
|
||||
if num_q_heads > _TRITON_SPARSE_MLA_MAX_HEADS:
|
||||
raise ValueError(
|
||||
"triton_sparse_mla is validated up to "
|
||||
f"{_TRITON_SPARSE_MLA_MAX_HEADS} query heads per rank; got "
|
||||
f"num_q_heads={num_q_heads}. Raise the attention TP degree or use "
|
||||
"another DSA prefill backend."
|
||||
)
|
||||
if index_kpool > 1:
|
||||
raise ValueError(
|
||||
"triton_sparse_mla does not support index_kpool > 1 (the pooled "
|
||||
f"indexer appends tail tokens to topk_indices); got {index_kpool}. "
|
||||
"Use flashmla_sparse, which reroutes the tail."
|
||||
)
|
||||
if union not in (0, 2, 4):
|
||||
raise ValueError(f"--dsa-triton-union must be 0, 2 or 4; got {union}.")
|
||||
if union:
|
||||
# tl.arange needs a power of two and tl.dot needs M >= 16; the tuned
|
||||
# union tiles stop at 32 rows.
|
||||
rows = num_q_heads * union
|
||||
if rows < 16 or rows > 32 or rows & (rows - 1):
|
||||
raise ValueError(
|
||||
f"--dsa-triton-union {union} needs num_q_heads * union to be a "
|
||||
f"power of two in [16, 32] (the union mma tile); got "
|
||||
f"num_q_heads={num_q_heads}, i.e. {rows} rows. Use a different "
|
||||
"group size or disable union."
|
||||
)
|
||||
|
||||
|
||||
def _resolve_dsa_triton_union(*, union: int, deterministic: bool) -> int:
|
||||
# Union makes a token's output depend on which tokens share its group and
|
||||
# on the T % G tail, which breaks the batch invariance deterministic
|
||||
# inference promises.
|
||||
if union and deterministic:
|
||||
logger.warning(
|
||||
"--dsa-triton-union %d is disabled under "
|
||||
"--enable-deterministic-inference: the union path makes a token's "
|
||||
"output depend on its batch neighbours.",
|
||||
union,
|
||||
)
|
||||
return 0
|
||||
return union
|
||||
|
||||
|
||||
class DeepseekSparseAttnBackend(
|
||||
DSAMetadataManagementMixin,
|
||||
DeepseekSparseAttnBackendKPoolMixin,
|
||||
@@ -407,6 +473,11 @@ class DeepseekSparseAttnBackend(
|
||||
self.supports_mha_one_shot: bool = True
|
||||
self.dsa_prefill_impl: _DSA_IMPL_T = get_exec().kernel.dsa_prefill_backend
|
||||
self.dsa_decode_impl: _DSA_IMPL_T = get_exec().kernel.dsa_decode_backend
|
||||
# Opt-in exact fast path for the Triton sparse-MLA prefill kernel.
|
||||
self.dsa_triton_union: int = _resolve_dsa_triton_union(
|
||||
union=get_exec().kernel.dsa_triton_union,
|
||||
deterministic=get_exec().deterministic.enable_deterministic_inference,
|
||||
)
|
||||
self.dsa_topk_backend: DSATopKBackend = DSATopKBackend.resolve(model_runner)
|
||||
if self.num_q_heads <= 64:
|
||||
self.flashmla_kv_num_q_heads = 64
|
||||
@@ -514,6 +585,14 @@ class DeepseekSparseAttnBackend(
|
||||
f"capability sm_{self.device_sm_major}x."
|
||||
)
|
||||
|
||||
if self.dsa_prefill_impl == "triton_sparse_mla":
|
||||
_validate_triton_sparse_mla_backend(
|
||||
device_sm_major=self.device_sm_major,
|
||||
num_q_heads=self.num_q_heads,
|
||||
union=self.dsa_triton_union,
|
||||
index_kpool=self.dsa_index_kpool,
|
||||
)
|
||||
|
||||
# `flashmla_sparse_q8` is prefill-only (FP8 decode goes through
|
||||
# `flashmla_kv`); reject it as a decode backend, since argparse accepts it
|
||||
# via the shared CLI choices.
|
||||
@@ -2128,7 +2207,11 @@ class DeepseekSparseAttnBackend(
|
||||
else None
|
||||
),
|
||||
)
|
||||
elif dsa_impl in ("flashmla_sparse", "flashmla_sparse_q8"):
|
||||
elif dsa_impl in (
|
||||
"flashmla_sparse",
|
||||
"flashmla_sparse_q8",
|
||||
"triton_sparse_mla",
|
||||
):
|
||||
if topk_transform_method == TopkTransformMethod.RAGGED:
|
||||
_has_prefix = any(forward_batch.extend_prefix_lens_cpu)
|
||||
page_table_1 = topk_indices
|
||||
@@ -2185,7 +2268,9 @@ class DeepseekSparseAttnBackend(
|
||||
layer_id=layer.layer_id,
|
||||
)
|
||||
|
||||
# bf16 path (dsa_impl == "flashmla_sparse").
|
||||
# bf16 path: `flashmla_sparse` and `triton_sparse_mla` take the
|
||||
# same inputs (concatenated q, dequantized bf16 KV, top-k list
|
||||
# as the slot table) and differ only in the kernel they call.
|
||||
if _has_prefix:
|
||||
page_table_1_flattened = (
|
||||
self.forward_metadata.page_table_1_flattened
|
||||
@@ -2199,6 +2284,15 @@ class DeepseekSparseAttnBackend(
|
||||
|
||||
if q_rope is not None:
|
||||
q_all = concat_mla_absorb_q_general(q_nope, q_rope)
|
||||
if dsa_impl == "triton_sparse_mla":
|
||||
return self._forward_triton_sparse_mla(
|
||||
q_all=q_all,
|
||||
kv_cache=kv_cache,
|
||||
page_table_1=page_table_1,
|
||||
sm_scale=layer.scaling,
|
||||
v_head_dim=layer.v_head_dim,
|
||||
topk_length=metadata.dsa_cache_seqlens_int32,
|
||||
)
|
||||
return self._forward_flashmla_sparse(
|
||||
q_all=q_all,
|
||||
kv_cache=kv_cache,
|
||||
@@ -2903,6 +2997,41 @@ class DeepseekSparseAttnBackend(
|
||||
o = o[:, :num_heads, :]
|
||||
return o
|
||||
|
||||
def _forward_triton_sparse_mla(
|
||||
self,
|
||||
q_all: torch.Tensor,
|
||||
kv_cache: torch.Tensor,
|
||||
v_head_dim: int,
|
||||
page_table_1: torch.Tensor,
|
||||
sm_scale: float,
|
||||
topk_length: Optional[torch.Tensor] = None,
|
||||
) -> torch.Tensor:
|
||||
from sglang.kernels.ops.attention.dsa.triton_sparse_mla_prefill import (
|
||||
sparse_mla_prefill,
|
||||
)
|
||||
|
||||
# Only the stream's own capture state gates union: the breakable prefill
|
||||
# graph runs attention eagerly between its segments while
|
||||
# get_is_capture_mode() stays set for the whole replay. Under real capture
|
||||
# the per-token path runs instead (same result); union is untested there.
|
||||
capturing = torch.cuda.is_current_stream_capturing()
|
||||
union = 0 if capturing else self.dsa_triton_union
|
||||
|
||||
# Same contract as `_forward_flashmla_sparse`: metadata rows are expected
|
||||
# to match q rows; fall back to full-width compute if they ever diverge.
|
||||
if topk_length is not None and topk_length.shape[0] != q_all.shape[0]:
|
||||
topk_length = None
|
||||
|
||||
return sparse_mla_prefill(
|
||||
q_all,
|
||||
kv_cache,
|
||||
page_table_1,
|
||||
sm_scale,
|
||||
v_head_dim,
|
||||
topk_length=topk_length,
|
||||
union=union,
|
||||
)
|
||||
|
||||
def _forward_flashinfer_sparse_mla(
|
||||
self,
|
||||
q_all: torch.Tensor,
|
||||
@@ -3672,8 +3801,11 @@ class DeepseekSparseAttnBackend(
|
||||
# flashmla_sparse_q8 shares flashmla_sparse's RAGGED prefill routing — the q8
|
||||
# dispatch lives inside the RAGGED branch of forward_extend; without this the
|
||||
# transform is PAGED, the q8 path is skipped, and the bf16 kernel crashes on
|
||||
# fp8 KV ("kv must have dtype kBFloat16").
|
||||
and self.dsa_prefill_impl in ("flashmla_sparse", "flashmla_sparse_q8")
|
||||
# fp8 KV ("kv must have dtype kBFloat16"). triton_sparse_mla is listed for
|
||||
# the same reason: its dispatch also dequantizes inside the RAGGED branch
|
||||
# and its kernel is bf16-only.
|
||||
and self.dsa_prefill_impl
|
||||
in ("flashmla_sparse", "flashmla_sparse_q8", "triton_sparse_mla")
|
||||
and forward_mode == ForwardMode.EXTEND
|
||||
):
|
||||
topk_transform_method = TopkTransformMethod.RAGGED
|
||||
|
||||
@@ -1209,6 +1209,13 @@ def dsa_impl_capability(impl: str) -> tuple[bool, str]:
|
||||
return False, f"aiter unavailable: {exc}"
|
||||
return True, ""
|
||||
|
||||
if impl == "triton_sparse_mla":
|
||||
if is_hip():
|
||||
return False, "triton_sparse_mla is a CUDA-only prefill backend"
|
||||
if major < 9:
|
||||
return False, f"{impl} requires SM>=9.0, got SM{major}.x"
|
||||
return True, ""
|
||||
|
||||
if impl == "flashmla_auto":
|
||||
# `flashmla_auto` resolves to flashmla_sparse / flashmla_kv at
|
||||
# forward time depending on `dsa_kv_cache_store_fp8`; both leaf
|
||||
@@ -1227,6 +1234,7 @@ DSA_PREFILL_IMPL_VARIANTS: tuple[str, ...] = (
|
||||
"tilelang",
|
||||
"trtllm",
|
||||
"aiter",
|
||||
"triton_sparse_mla",
|
||||
)
|
||||
DSA_DECODE_IMPL_VARIANTS: tuple[str, ...] = (
|
||||
"flashmla_sparse",
|
||||
@@ -1243,7 +1251,7 @@ DSA_DECODE_IMPL_VARIANTS: tuple[str, ...] = (
|
||||
# take in FP8 deployments. The `flashmla_kv` decode kernel and *both*
|
||||
# flashmla prefill kernels are the production-relevant FP8 paths.
|
||||
DSA_FP8_COMPATIBLE_PREFILL_IMPLS: frozenset[str] = frozenset(
|
||||
{"flashmla_sparse", "flashmla_kv", "flashmla_auto"}
|
||||
{"flashmla_sparse", "flashmla_kv", "flashmla_auto", "triton_sparse_mla"}
|
||||
)
|
||||
DSA_FP8_COMPATIBLE_DECODE_IMPLS: frozenset[str] = frozenset(
|
||||
{"flashmla_kv", "flashmla_auto"}
|
||||
|
||||
Reference in New Issue
Block a user