[Perf] [AMD] dsv4: fp8 unified_kv decode/verify on FlyDSL v4 nm decode with a latency-model split plan (1/8) (#44016)

Co-authored-by: Duyi-Wang <duyi.wang@amd.com>
This commit is contained in:
kk
2026-10-16 01:50:16 -07:00
committed by GitHub
co-authored by Duyi-Wang
parent 9baace8456
commit ab5858ac8a
15 changed files with 7994 additions and 33 deletions
@@ -277,6 +277,19 @@ for _mod, _fn, _backend, _device in [
KernelBackend.FLYDSL,
CapabilityRequirement.HIP,
),
# TODO(aiter#5882): remove once ROCm/aiter#5882 is in the aiter pin
(
"dsv4.flydsl_v4nm.mla_v4_decode",
"flydsl_mla_decode_fwd_v4_nm",
KernelBackend.FLYDSL,
CapabilityRequirement.HIP,
),
(
"dsv4.flydsl_v4nm.mla_v4_decode",
"flydsl_mla_v4_decode_fused",
KernelBackend.FLYDSL,
CapabilityRequirement.HIP,
),
]:
register_kernel(
KernelSpec(
@@ -0,0 +1,41 @@
# TODO(aiter#5882): remove once ROCm/aiter#5882 is in the aiter pin
"""In-tree copy of ROCm/aiter#5882: DeepSeek-V4 fp8 MLA decode (v4 nm contract)
in one FlyDSL launch on gfx950, for aiter pins that do not ship it yet.
Vendored from ROCm/aiter#5882 at commit 03359c43df (the head adapted to
ROCm/aiter#6377); see the per-file headers. The entry points match the ones
the PR adds to ``aiter.ops.flydsl``. ``unified_kv_kernels.runtime`` prefers
aiter's own copy and only falls back to this one (SGLANG_DSV4_FLYDSL_DECODE_SOURCE).
Needs an aiter with ROCm/aiter#6377 (typed ``_to_ptr_global``): importing this
package raises ImportError on an older aiter, so callers fall back to asm.
"""
import inspect
# AITER owns the FlyDSL toolchain bootstrap and the shared tensor/buffer shims.
# Import it before the local kernel modules so its FlyDSL setup is active.
import aiter as _aiter # noqa: F401
from aiter.ops.flydsl.kernels import communication_ops_utils as _cu
if "dtype" not in inspect.signature(_cu._to_ptr_global).parameters:
raise ImportError(
"the in-tree FlyDSL v4 nm decode (ROCm/aiter#5882) needs an aiter with "
"ROCm/aiter#6377 (typed communication_ops_utils._to_ptr_global)"
)
from .mla_v4_decode import ( # noqa: E402
flydsl_mla_decode_fwd_v4_nm,
flydsl_mla_v4_decode_fused,
flydsl_mla_v4_decode_supported,
flydsl_mla_v4_grouped_pays,
)
from .mla_v4_grouped import flydsl_mla_v4_grouped_verify_meta # noqa: E402
__all__ = [
"flydsl_mla_decode_fwd_v4_nm",
"flydsl_mla_v4_decode_fused",
"flydsl_mla_v4_decode_supported",
"flydsl_mla_v4_grouped_pays",
"flydsl_mla_v4_grouped_verify_meta",
]
@@ -0,0 +1,2 @@
# TODO(aiter#5882): remove once ROCm/aiter#5882 is in the aiter pin
"""FlyDSL kernel bodies of the in-tree ROCm/aiter#5882 copy (see the package)."""
@@ -0,0 +1,252 @@
# SPDX-License-Identifier: MIT
# Copyright (C) 2024-2026, Advanced Micro Devices, Inc. All rights reserved.
#
# Vendored from ROCm/aiter#5882 (https://github.com/ROCm/aiter/pull/5882),
# commit 03359c43df (03359c43df40abf91ac38a2040c9b8e284d92297),
# file aiter/ops/flydsl/kernels/mla_v4_decode_common.py.
# Unchanged except for import paths and sglang formatting (isort, ruff format).
# TODO(aiter#5882): remove once ROCm/aiter#5882 is in the aiter pin
"""Shared helpers of the FlyDSL v4 nm MLA decode (mla_v4_decode_gfx950.py): math
intrinsics, cross-lane butterflies, the global atomics / loads / fences of the
cross-split merge protocol, and the mxfp8 (ceil-e8m0) block scale."""
import flydsl.compiler as flyc
import flydsl.expr as fx
from aiter.ops.flydsl.kernels import communication_ops_utils as cu
from flydsl._mlir.dialects import llvm as _llvm_d
from flydsl.expr import arith as _arith
from flydsl.expr.typing import T
FP8_MAX = 448.0
EX_FLOOR = 94 # smallest e8m0 block scale emitted: 2^(94 - 127)
def exp2(x):
return fx.Float32(fx.rocdl.exp2(T.f32, fx.Float32(x).ir_value()))
def rcp(x):
return fx.Float32(fx.rocdl.rcp(T.f32, fx.Float32(x).ir_value()))
def ldexp(x, e):
return fx.Float32(
_llvm_d.call_intrinsic(
T.f32,
"llvm.ldexp.f32.i32",
[fx.Float32(x).ir_value(), fx.Int32(e).ir_value()],
[],
[],
)
)
def fabs_f32(x):
return fx.Float32(
_llvm_d.call_intrinsic(
T.f32, "llvm.fabs.f32", [fx.Float32(x).ir_value()], [], []
)
)
def maxnum_f32(a, b):
return fx.Float32(
_llvm_d.call_intrinsic(
T.f32,
"llvm.maxnum.f32",
[fx.Float32(a).ir_value(), fx.Float32(b).ir_value()],
[],
[],
)
)
def fabs_bits(x):
return fabs_f32(x).bitcast(fx.Int32)
def udiv(a, b):
return fx.Int32(_arith.divui(fx.Int32(a).ir_value(), fx.Int32(b).ir_value()))
def urem(a, b):
return fx.Int32(_arith.remui(fx.Int32(a).ir_value(), fx.Int32(b).ir_value()))
def smax_i32(a, b):
return (a > b).select(a, b)
def smin_i32(a, b):
return (a < b).select(a, b)
def dpp_i32(x, ctrl):
return fx.Int32(
fx.rocdl.update_dpp(
T.i32, fx.Int32(0).ir_value(), fx.Int32(x).ir_value(), ctrl, 0xF, 0xF, True
)
)
def plswap(x, width, op2):
st = _llvm_d.StructType.get_literal([T.i32, T.i32])
op = fx.rocdl.permlane16_swap if width == 16 else fx.rocdl.permlane32_swap
r = op(st, fx.Int32(x).ir_value(), fx.Int32(x).ir_value(), False, False)
a = fx.Int32(_llvm_d.extractvalue(T.i32, r, [0]))
b = fx.Int32(_llvm_d.extractvalue(T.i32, r, [1]))
return op2(a, b)
_DPP_XOR = {1: 0xB1, 2: 0x4E, 4: 0x141, 8: 0x128}
def butterfly_i32(x, nlanes, op2, first=1):
k = first
while k < nlanes * first:
if k <= 8:
x = op2(x, dpp_i32(x, _DPP_XOR[k]))
else:
x = plswap(x, k, op2)
k *= 2
return x
def mxfp8_exp(amb):
# e8m0 of ceil(amax / 448) from amax's bits: 448 = 1.75 * 2^8, so a mantissa
# above 0.75 (0x600000) rounds the exponent up
ea = amb >> fx.Int32(23)
ex = (
ea
- fx.Int32(8)
+ ((amb & fx.Int32(0x7FFFFF)) > fx.Int32(0x600000)).select(
fx.Int32(1), fx.Int32(0)
)
)
ex = (ex < fx.Int32(EX_FLOOR)).select(fx.Int32(EX_FLOOR), ex)
return (ea >= fx.Int32(255)).select(fx.Int32(254), ex)
def fp8_scaled(y, nexp):
v = ldexp(y, nexp).maximumf(fx.Float32(-FP8_MAX))
return (v > fx.Float32(FP8_MAX)).select(fx.Float32(FP8_MAX), v)
def xcc_id():
return fx.Int32(
_llvm_d.InlineAsmOp(
T.i32,
[],
"s_getreg_b32 $0, hwreg(HW_REG_XCC_ID)",
"=s",
has_side_effects=True,
).res
)
def pin_v64(x):
return fx.Int64(
_llvm_d.InlineAsmOp(
T.i64, [fx.Int64(x).ir_value()], "; pin $0", "=v,0", has_side_effects=True
).res
)
def gload(addr_i64, n):
return fx.generic_load(cu._to_ptr_global(addr_i64, fx.Float32, 4 * n), count=n)
def inv_l1():
_llvm_d.InlineAsmOp(None, [], "buffer_inv sc0", "", has_side_effects=True)
def rmw_add64_agent(addr_i64, val):
return fx.atomic_add(
cu._to_ptr_global(addr_i64, fx.Int64, 8), fx.Int64(val), syncscope="agent"
)
def ld64_agent(addr_i64, volatile=True):
return fx.generic_load(
cu._to_ptr_global(addr_i64, fx.Int64, 8),
volatile=volatile,
memory_order=fx.AtomicOrdering.Monotonic,
syncscope=fx.rocdl.SyncScope.AgentOneAs,
)
def st_raw(addr_i64, value, align):
fx.generic_store(cu._to_ptr_global(addr_i64, alignment=align), value)
def st_agent(addr_i64, value, align):
fx.generic_store(
cu._to_ptr_global(addr_i64, alignment=align),
value,
memory_order=fx.AtomicOrdering.Monotonic,
syncscope=fx.rocdl.SyncScope.AgentOneAs,
)
def st_agent_vec(addr_i64, value, nbytes):
# sc1 store: written through to the device coherence point (no L2 write-back
# needed); s_nop 1 covers the store-data VGPR hazard the backend does not see
if nbytes == 4:
v = fx.Float32(value).bitcast(fx.Int32)
op = "global_store_dword $0, $1, off sc1\n\ts_nop 1"
else:
assert nbytes == 16, nbytes
v = fx.Vector(value).bitcast(fx.Int32)
op = "global_store_dwordx4 $0, $1, off sc1\n\ts_nop 1"
_llvm_d.InlineAsmOp(
None,
[fx.Int64(addr_i64).ir_value(), _arith.unwrap(v)],
op,
"v,v,~{memory}",
has_side_effects=True,
)
def ld_agent_f32(addr_i64, n):
if n == 1:
return ld_agent_i32(addr_i64, volatile=False).bitcast(fx.Float32)
parts = [
ld64_agent(addr_i64 + fx.Int64(8 * i), volatile=False) for i in range(n // 2)
]
return fx.Vector.from_elements(parts, fx.Int64).bitcast(fx.Float32)
@flyc.jit
def spin_all_arrived(addr_i64, full, bit):
cur = ld64_agent(addr_i64)
while ((cur & full) != full) & ((cur & bit) != fx.Int64(0)):
cur = ld64_agent(addr_i64)
return cur
def ld_agent_i32(addr_i64, volatile=True):
return fx.generic_load(
cu._to_ptr_global(addr_i64),
volatile=volatile,
memory_order=fx.AtomicOrdering.Monotonic,
syncscope=fx.rocdl.SyncScope.AgentOneAs,
)
def tickets_state(v, want):
z = fx.Int64(fx.rocdl.ballot(T.i64, (v == fx.Int32(0)).ir_value()))
o = fx.Int64(fx.rocdl.ballot(T.i64, (v != want).ir_value()))
return (z == fx.Int64(0)).select(fx.Int32(1), fx.Int32(0)) | (
(o == fx.Int64(0)).select(fx.Int32(2), fx.Int32(0))
)
@flyc.jit
def spin_tickets_state(addr_lane_i64, v, max_polls, want):
r = tickets_state(v, want)
k = fx.Int32(0)
while ((r & fx.Int32(1)) == fx.Int32(0)) & (k < fx.Int32(max_polls)):
r = tickets_state(ld_agent_i32(addr_lane_i64), want)
k = k + fx.Int32(1)
return r
@@ -0,0 +1,718 @@
# SPDX-License-Identifier: MIT
# Copyright (C) 2024-2026, Advanced Micro Devices, Inc. All rights reserved.
#
# Vendored from ROCm/aiter#5882 (https://github.com/ROCm/aiter/pull/5882),
# commit 03359c43df (03359c43df40abf91ac38a2040c9b8e284d92297),
# file aiter/ops/flydsl/mla_v4_decode.py.
# Unchanged except for import paths and sglang formatting (isort, ruff format).
# TODO(aiter#5882): remove once ROCm/aiter#5882 is in the aiter pin
"""FlyDSL DeepSeek-V4 fp8 MLA decode (v4 nm contract) on gfx950.
flydsl_mla_decode_fwd_v4_nm: drop-in for aiter.mla.mla_decode_fwd_v4_nm (bf16 output).
flydsl_mla_v4_decode_fused: the same attention + inverse RoPE + wo_a mxfp8 quant.
The split plan is keyed by the batch size, head count, max_seqlen_q and a stream hint,
never by kv lengths (graph-capture safe). The merge scratch is owned here, one per
(device, stream).
"""
from __future__ import annotations
import dataclasses
from functools import cache
import flydsl.expr as fx
import torch
from aiter.jit.utils.chip_info import get_gfx
from aiter.ops.flydsl.kernels.tensor_shim import _run_compiled, ptr_arg
from .kernels.mla_v4_decode_gfx950 import (
DV,
MAX_GROUPS,
MAX_S,
ROPE,
ROWB,
WS_HDR,
V4Cfg,
build_mla_v4_decode,
)
from .mla_v4_grouped import GROUPED_MAX_MSQ
__all__ = [
"flydsl_mla_decode_fwd_v4_nm",
"flydsl_mla_v4_decode_fused",
"flydsl_mla_v4_decode_supported",
"flydsl_mla_v4_grouped_pays",
]
# (H, max_seqlen_q) pairs the asm v4 nm kernel ships
SUPPORTED = ((16, 1), (16, 2), (16, 4), (32, 1), (64, 1), (128, 1))
# Split plans measured on MI355X (cold caches, vs the best asm split count + merge).
# H16 msq1 with a stream hint: T_pad upper bound -> (CSA, HCA) splits (HCA past
# T_pad 40: _plan_h16's key-balanced grid)
_POLICY_H16 = {
8: (18, 18),
16: (12, 12),
24: (9, 9),
32: (8, 8),
40: (6, 6),
48: (4, 4),
64: (4, 4),
80: (3, 3),
128: (2, 2),
160: (3, 1),
256: (1, 1),
}
# below _POLICY_Q16: B upper bound -> (16-row blocks per CTA, splits) of K SPREAD plans
_POLICY_WIDE = {
(16, 2): ((1, 1, 32), (None, 1, 16)),
(16, 4): ((None, 1, 16),),
(32, 1): ((1, 1, 24), (None, 1, 16)),
(64, 1): ((None, 1, 16),),
(128, 1): ((None, 1, 16),),
}
# stream-length bodies (V4Cfg.SUB): B upper bound -> longest short stream (K, S = 1);
# (H, msq) -> ((B max, B min, ((RB, S, longest stream), ...)), ...) mid-length streams
_SHORT_KEYS = ((8, 192), (16, 256), (64, 768), (128, 256))
# Q16 mains: 32-row ones take the short body up to _Q16_SHORT_KEYS keys; 64-row ones
# past B 16 at S >= 3 run streams of <= max(_Q16_RB2_KEYS / S, _Q16_RB2_MIN) keys on
# unsplit 32-row Q16 CTAs (at S = 2 the joint kernel spills: long streams pay more)
_Q16_SHORT_KEYS = 160
_Q16_RB2_KEYS, _Q16_RB2_MIN = 2304, 448
_POLICY_MID = {
(16, 2): ((1, 1, ((1, 16, 2048),)), (16, 9, ((1, 8, 4096),))),
(16, 4): ((8, 5, ((1, 8, 3072),)), (16, 9, ((1, 4, 2048),))),
(32, 1): ((16, 9, ((1, 8, 4096),)),),
(64, 1): ((8, 5, ((1, 8, 3072),)), (16, 9, ((1, 4, 2048),))),
}
# smallest B of each pair on Q16 CTAs
_POLICY_Q16 = {(16, 2): 9, (16, 4): 5, (32, 1): 9, (64, 1): 5, (128, 1): 2}
# grids a little past half the GPU run 2 splits: largest grid, in 1/32 of the CUs, for
# Q16 (RB, NRG) and for the H16 K layout
_PAST_HALF = {(4, 1): 18, (2, 1): 19}
_PAST_HALF_K = 19
# Q16 split count -> streams of at most this many 32-key tiles run unsplit (SHORT)
_Q16_SHORT_TILES = {2: 24, 3: 10}
# unsplit Q16 grids of fewer CTAs than CUs: split 1 of the longest streams of > _IDLE_F
# tiles on the idle CUs (V4Cfg.IDLE), past them on _IDLE_X2 more CTAs per XCD for
# streams of > _IDLE_W tiles; larger grids of two row groups (up to 512 sequences):
# splits 1-3 of streams of > _IDLE_WL tiles on _IDLE_XL CTAs per XCD past the grid
_IDLE_X2, _IDLE_F, _IDLE_W = 4, 32, 96
_IDLE_XL, _IDLE_WL = 16, 64
# PV16R key window: (lo, hi at B <= CUs, hi above)
_PV16_LO, _PV16_HI = 256, (4096, 2048)
# grouped HCA: streams of <= 17 32-key tiles on the K body, longer ones on RQ
_GROUPED_RQ_TILES = 18
# H16 bf16 without a hint: most splits per sequence; longest unsplit stream; shortest
# key-balanced SK run in 64-key tiles (grid of one, two CTAs per CU)
_H16_MAX_S = 18
_H16_SHORT_KEYS = 128
_H16_SKT = (8, 12)
# SK grids of at most this many splits per sequence: V4Cfg.TAIL where it adds some
_H16_TAIL_SKD = 8
# 64-row Q16 with fewer splits than this per sequence: V4Cfg.TAIL where it adds some
_TAIL_S = 16
@cache
def _num_cus(device_index: int) -> int:
return torch.cuda.get_device_properties(device_index).multi_processor_count
def flydsl_mla_v4_decode_supported(
num_heads: int, max_seqlen_q: int = 1, grouped: bool = False
) -> bool:
"""True if the FlyDSL v4 decode covers (num_heads, max_seqlen_q) on this GPU;
grouped: with q_kv_bounds (H = 16, max_seqlen_q <= 8)."""
try:
if get_gfx() != "gfx950":
return False
except Exception: # noqa: BLE001
return False
if grouped:
return num_heads == 16 and 1 <= max_seqlen_q <= GROUPED_MAX_MSQ
return (num_heads, max_seqlen_q) in SUPPORTED
def flydsl_mla_v4_grouped_pays(
num_heads: int,
num_draft: int,
num_requests: int,
hint: str | None = "hca",
epi: str = "invrope_mxfp8",
) -> bool:
"""Whether a DSpark verify step should share one stream per request (grouped
q_kv_bounds) on its HCA / SWA-only layers (MI355X, DSv4-Pro verify step)."""
if num_heads != 16 or not 1 <= num_draft <= GROUPED_MAX_MSQ:
return False
if epi == "bf16" and hint in (None, "hca"):
return num_requests >= 32
return num_requests >= 64
def _plan(
H: int,
msq: int,
B: int,
epi: str = "bf16",
hint: str | None = None,
cus: int = 256,
grouped: bool = False,
) -> V4Cfg:
"""Variant for B sequences. hint: "csa" | "hca" | "swa" (SWA-only layer) | None."""
cfg = _plan_for(H, msq, max(1, B), epi, hint, cus, grouped)
if cfg.ws_need(B)[0] > MAX_GROUPS:
# more merge groups than protocol slots: run unsplit
cfg = dataclasses.replace(
cfg,
S=1,
ADAPT=0,
MINT=1,
SUB=(),
SPREAD=False,
SHORT=0,
SHORTM=False,
SHORTF=0,
OWN=False,
SK=0,
SKMEAN=False,
IDLE=(),
)
return cfg
def _plan_for(H, msq, B, epi, hint, cus, grouped) -> V4Cfg:
if grouped:
return _plan_grouped(msq, B, epi, hint, cus)
if H == 16 and msq == 1:
return _plan_h16(B, epi, hint, cus)
if B >= _POLICY_Q16[(H, msq)]:
return _with_subs(_q16_plan(H, msq, B, epi, cus), B, cus)
for bmax, rb, s in _POLICY_WIDE[(H, msq)]:
if bmax is None or B <= bmax:
break
cfg = V4Cfg(H=H, MSQ=msq, RB=rb, S=s, EPI=epi, KC=True, SPREAD=True)
return _with_subs(cfg, B, cus)
def _plan_h16(B, epi, hint, cus) -> V4Cfg:
tpad = -(-B // 8) * 8
adapt = {}
if hint == "swa":
s = 1
elif epi == "bf16" and hint is None:
s = min(_H16_MAX_S, max(1, cus // tpad))
if 16 <= tpad <= 2 * cus:
# ragged batches: the key-balanced grid (V4Cfg.SK; past cus sequences two
# CTAs per CU)
sk = cus * -(-tpad // cus)
cfg = _sk_plan(B, epi, cus, 16 * cus // sk, SK=sk, SKT=_H16_SKT[sk > cus])
if 2 * tpad > cus and tpad <= cus * _PAST_HALF_K // 32:
cfg = dataclasses.replace(cfg, SKD=2, SKT=14, SKMEAN=True, SHORT=4)
return cfg
if tpad > cus:
s = 2
adapt = {"ADAPT": cus, "MINT": 16}
elif tpad < 16:
short = V4Cfg(H=16, RB=1, EPI=epi, KC=True)
adapt = {"SUB": ((_H16_SHORT_KEYS, short),)}
else:
s = 1
for tb, v in _POLICY_H16.items():
if tpad <= tb:
s = v[0 if hint == "csa" else 1]
break
if hint == "hca" and tpad > 40:
# ragged HCA batches (a few long sessions set the longest stream): the
# key-balanced grid; past one sequence per CU, run-time split counts
if tpad > cus:
return V4Cfg(H=16, RB=1, S=8, EPI=epi, KC=True, ADAPT=cus, MINT=4)
return _sk_plan(B, epi, cus, 8, SK=cus, SKT=6)
return V4Cfg(H=16, RB=1, S=min(s, MAX_S), EPI=epi, KC=True, **adapt)
def _sk_plan(B, epi, cus, S, **kw) -> V4Cfg:
"""H16 key-balanced grid (V4Cfg.SK) of at most max(S, SKD) splits, else SKD
splits per sequence: cus // T_pad, or cus // B on V4Cfg.TAIL where that is more"""
tpad = -(-B // 8) * 8
skd = max(1, min(_H16_MAX_S, cus // tpad))
tail = skd < cus // B <= _H16_TAIL_SKD
skd = cus // B if tail else skd
return V4Cfg(
H=16,
RB=1,
EPI=epi,
KC=True,
S=max(S, skd),
SKD=skd,
TAIL=tail,
MINT=3 if 4 * tpad > cus else 1,
SHORT=_H16_SHORT_KEYS // 64,
**kw,
)
def _with_subs(cfg: V4Cfg, B: int, cus: int) -> V4Cfg:
"""cfg + its stream-length bodies where they fit (at most the sequence's CTAs; the
short body at most one CTA per CU over the batch; under Q16 16-row K or 32-row Q16
bodies)."""
if cfg.S == 1 or cfg.XR or cfg.IDLE:
return cfg
if cfg.Q16 and cfg.RB == 4 and B > 16 and cfg.S >= 3:
rb2 = dataclasses.replace(cfg, RB=2, S=1, SHORT=0, PF16=False)
keys = max(_Q16_RB2_KEYS // cfg.S, _Q16_RB2_MIN)
# (the main's short streams all take the body)
return dataclasses.replace(cfg, SHORT=0, SUB=((keys, rb2),))
subs = []
keys = next((k for bmax, k in _SHORT_KEYS if B <= bmax), 0)
if cfg.Q16 and cfg.RB == 2:
keys = min(keys, _Q16_SHORT_KEYS)
rbs = (1,) if cfg.Q16 else (1, 2)
for rb in rbs if keys else ():
nrg = cfg.H * cfg.MSQ // (16 * rb)
if nrg <= cfg.NRG * cfg.S and B * nrg <= cus:
subs.append((keys, _k_body(cfg, rb, 1)))
break
for bmax, bmin, mids in _POLICY_MID.get((cfg.H, cfg.MSQ), ()):
if bmin <= B <= bmax:
for rb, s, mkeys in mids:
sub = _k_body(cfg, rb, s)
fits = rb in rbs and sub.NRG * s <= cfg.NRG * cfg.S and sub != cfg
if fits and mkeys > keys:
subs.append((mkeys, sub))
keys = mkeys
break
return dataclasses.replace(cfg, SUB=tuple(subs)) if subs else cfg
def _q16_plan(H: int, msq: int, B: int, epi: str, cus: int) -> V4Cfg:
rb = 4 if H * msq % 64 == 0 else 2
nrg = H * msq // (16 * rb)
bpad = -(-B // 8) * 8
s = max(1, min(MAX_S, cus // (bpad * nrg)))
xr = nrg > 1 and bpad <= 8
if nrg > 1 and bpad > 8:
# row groups on consecutive XCDs where that allows more splits
s_xr = min(MAX_S, cus // 8 // -(-B * nrg // 8))
if s_xr > s:
s, xr = s_xr, True
# the units past the last complete XCD octet over all XCDs (V4Cfg.TAIL)
tail_ok = rb == 4 and s < _TAIL_S and 8 < B < cus // 8 * nrg and B % 8
s_tl = _tail_splits(B, nrg, xr, cus) if tail_ok else 0
tail, s = s_tl > s, max(s, s_tl)
mean_short = rb == 4 and nrg == 1 and 2 * bpad == cus
past_half = (
s == 1
and not xr
and cus < 2 * bpad * nrg
and bpad * nrg <= cus * _PAST_HALF.get((rb, nrg), 0) // 32
)
if past_half:
s = 2
if s == 1 and not xr and (bpad * nrg < cus or nrg > 1 and bpad <= 512):
s, idle = 2, (cus // 8, _IDLE_X2, _IDLE_F, _IDLE_W)
if bpad * nrg >= cus:
s, idle = 4, (bpad // 8 * nrg, _IDLE_XL, _IDLE_F, _IDLE_WL)
return V4Cfg(
H=H,
MSQ=msq,
RB=rb,
S=s,
EPI=epi,
LAYOUT="RD",
Q16=True,
PF16=True,
PFN=cus if bpad * nrg > cus else 0,
IDLE=idle,
)
if s == 1:
short = 0
elif mean_short:
short = 64
elif past_half:
short = 1 << 16
elif bpad <= 8:
short = 6
else:
short = _Q16_SHORT_TILES.get(s, 8)
cfg = V4Cfg(
H=H,
MSQ=msq,
RB=rb,
S=s,
EPI=epi,
LAYOUT="RD",
Q16=True,
SHORT=short,
SHORTM=(mean_short or past_half) and s > 1,
SHORTF=16 if past_half else 0,
PF16=s > 1,
XR=xr,
PFN=cus if bpad * nrg > cus else 0,
TAIL=tail,
)
if rb == 4 and nrg == 1 and bpad >= cus and epi == "bf16":
cfg = dataclasses.replace(
cfg, PV16R=(_PV16_LO, _PV16_HI[0] if bpad <= cus else _PV16_HI[1])
)
return cfg
def _tail_splits(B: int, nrg: int, xr: bool, cus: int) -> int:
# most splits under V4Cfg.TAIL with at most one CTA per CU on every XCD (one row
# group: 7/8 of them; a full XCD costs up to 1.25x at 1k-4k keys; XR: at most 12,
# more cost ~8 % at 200-500 keys)
units, per = (B * nrg, 1) if xr else (B, nrg)
cap = cus // 8 if nrg > 1 else cus * 7 // 64
s = 12 if xr else MAX_S
while s > 1 and units // 8 * s * per + -(-(units % 8) * s * per // 8) > cap:
s -= 1
return s
def _k_body(cfg: V4Cfg, rb: int, s: int) -> V4Cfg:
return V4Cfg(
H=cfg.H,
MSQ=cfg.MSQ,
RB=rb,
S=s,
EPI=cfg.EPI,
KC=True,
SPREAD=cfg.SPREAD,
TAIL=cfg.TAIL,
)
def _plan_grouped(msq: int, B: int, epi: str, hint: str | None, cus: int) -> V4Cfg:
"""Grouped verify at H = 16: B groups of up to msq drafts sharing one stream."""
hca = hint in (None, "hca")
if epi == "bf16" and (B > 32 or (B == 32 and hca)):
return _grouped_q16(msq, B, hca, cus)
if B >= 32 and hca:
# long streams on one 128-row RQ CTA per (group, split), short ones on K
short = V4Cfg(
H=16,
MSQ=GROUPED_MAX_MSQ,
RB=2 if B >= 64 else 1,
EPI=epi,
KC=True,
MASK="bounds",
)
return V4Cfg(
H=16,
MSQ=GROUPED_MAX_MSQ,
RB=GROUPED_MAX_MSQ,
S=4,
EPI=epi,
LAYOUT="RQ",
MASK="bounds",
OWN=True,
SUB=((32 * (_GROUPED_RQ_TILES - 1), short),),
)
if B > 32:
return V4Cfg(
H=16,
MSQ=msq + (msq & 1),
RB=2,
EPI=epi,
KC=hint != "swa",
MASK="bounds",
)
s = 1 if hint == "swa" or B > 16 else (2 if B > 8 else 4)
return V4Cfg(H=16, MSQ=msq, RB=1, S=s, EPI=epi, MASK="bounds")
def _grouped_q16(msq: int, B: int, hca: bool, cus: int) -> V4Cfg:
"""32-row Q16 CTAs (two drafts each), 64-row ones where the 32-row grid exceeds
one CTA per CU and they need fewer."""
mp = msq + (msq & 1)
rb = 2
nrg = 16 * mp // 32
bpad = -(-B // 8) * 8
mp4 = -(-msq // 4) * 4
if bpad * nrg > cus and 16 * mp4 // 64 < nrg:
mp, rb, nrg = mp4, 4, 16 * mp4 // 64
s = max(1, min(4, cus // (bpad * nrg))) if hca else 1
return V4Cfg(
H=16,
MSQ=mp,
RB=rb,
S=s,
LAYOUT="RD",
MASK="bounds",
Q16=True,
SHORT=16 if s > 1 else 0,
PF16=s > 1,
)
class _Workspace:
"""Grow-only merge scratch of one (device, stream): protocol slots (hdr, zero
between launches) + split partials (buf). A new stream takes a spare hdr zeroed
at an earlier eager launch, so a graph captured on it replays no memset. Old
buffers stay alive: a captured graph may still reference them."""
def __init__(self, device):
self.device = device
self.hdr = None
self.buf = None
self._keep = []
def ensure(self, nbytes: int):
if self.hdr is None:
self.hdr = _HDR_SPARE.pop(self.device.index, None)
if self.hdr is None:
self.hdr = torch.zeros(
WS_HDR // 4, dtype=torch.int32, device=self.device
)
if (
not torch.cuda.is_current_stream_capturing()
and self.device.index not in _HDR_SPARE
):
_HDR_SPARE[self.device.index] = _zeroed_hdr(self.device)
need = -(-nbytes // 4)
if self.buf is None or self.buf.numel() < need:
if self.buf is not None:
self._keep.append(self.buf)
need = max(need, self.buf.numel() * 2)
self.buf = torch.empty(need, dtype=torch.int32, device=self.device)
return self
_WORKSPACES: dict = {}
_HDR_SPARE: dict = {}
_ZERO_STREAMS: dict = {}
_COMPILED: set = set()
_DUMMY: dict = {}
def _zeroed_hdr(device):
# zeroed on an idle side stream, before any stream can use it
zs = _ZERO_STREAMS.get(device.index)
if zs is None:
zs = _ZERO_STREAMS[device.index] = torch.cuda.Stream(device)
with torch.cuda.stream(zs):
hdr = torch.zeros(WS_HDR // 4, dtype=torch.int32, device=device)
zs.synchronize()
return hdr
def _workspace(device) -> _Workspace:
stream = torch.cuda.current_stream(device)
key = (device.index, stream.cuda_stream)
ws = _WORKSPACES.get(key)
if ws is None:
ws = _WORKSPACES[key] = _Workspace(device)
return ws
def _launch(
cfg: V4Cfg,
B,
q,
q_rope,
kv,
kv_rope,
qo_indptr,
kv_indptr,
kv_indices,
sink,
out,
out_s0,
out_s1,
xs=None,
positions=None,
freqs=None,
bounds=None,
):
device = q.device
dmy = _DUMMY.get(device.index)
if dmy is None:
dmy = _DUMMY[device.index] = torch.zeros(64, dtype=torch.int64, device=device)
groups, nbytes = cfg.ws_need(B)
if groups:
ws = _workspace(device).ensure(nbytes)
hdr, wsb = ws.hdr, ws.buf
else:
hdr = wsb = dmy
if cfg not in _COMPILED and torch.cuda.is_current_stream_capturing():
raise RuntimeError(
f"flydsl_mla_v4_decode: variant {cfg} not compiled; run this batch size "
"once eagerly before graph capture"
)
u8 = torch.uint8
epi = cfg.EPI == "invrope_mxfp8"
_run_compiled(
build_mla_v4_decode(cfg),
ptr_arg(q.view(u8), fx.Uint8),
ptr_arg(q_rope, fx.BFloat16),
ptr_arg(kv.view(u8), fx.Uint8),
ptr_arg(kv_rope, fx.BFloat16),
ptr_arg(dmy if qo_indptr is None else qo_indptr, fx.Int32),
ptr_arg(kv_indptr, fx.Int32),
ptr_arg(kv_indices, fx.Int32),
ptr_arg(sink, fx.Float32),
ptr_arg(out.view(u8) if epi else out, fx.Uint8 if epi else fx.BFloat16),
ptr_arg(dmy if xs is None else xs.view(u8), fx.Uint8),
ptr_arg(dmy if positions is None else positions, fx.Int64),
ptr_arg(dmy if freqs is None else freqs, fx.Float32),
ptr_arg(wsb, fx.Int32),
ptr_arg(dmy if bounds is None else bounds, fx.Int32),
ptr_arg(hdr, fx.Int32),
B,
kv.shape[0],
out_s0,
out_s1,
fx.Stream(torch.cuda.current_stream(device)),
)
_COMPILED.add(cfg)
def _check_bounds(q_kv_bounds, total_q):
if q_kv_bounds is not None:
assert q_kv_bounds.dtype == torch.int32 and q_kv_bounds.is_contiguous()
assert q_kv_bounds.dim() == 2 and q_kv_bounds.shape[1] == 4
assert q_kv_bounds.shape[0] >= total_q, (q_kv_bounds.shape, total_q)
def flydsl_mla_decode_fwd_v4_nm(
q,
qrope,
kv_buffer,
kvrope,
output,
qo_indptr,
kv_indptr,
kv_page_indices,
max_seqlen_q,
*,
sink,
hint=None,
q_kv_bounds=None,
):
"""aiter.mla.mla_decode_fwd_v4_nm on gfx950: writes the final bf16 attention into
``output`` [total_q, H, 512] (rows without keys become NaN).
hint: "csa" | "hca" | "swa" | None, only selects the split plan.
q_kv_bounds: optional int32 [total_q, 4] (a0, a1, b0, b1); q row r attends stream
position j of its sequence iff a0 <= j < a1 or b0 <= j < b1 (grouped verify).
"""
H = q.shape[1]
msq = int(max_seqlen_q)
grouped = q_kv_bounds is not None
if not flydsl_mla_v4_decode_supported(H, msq, grouped):
raise RuntimeError(
f"flydsl_mla_decode_fwd_v4_nm: no variant for num_heads={H}, "
f"max_seqlen_q={msq}, grouped={grouped}"
)
B = qo_indptr.numel() - 1
assert q.dim() == 3 and q.shape[2] == ROWB and q.element_size() == 1, q.shape
assert qrope.shape[1:] == (H, ROPE) and qrope.dtype == torch.bfloat16
assert q.is_contiguous() and qrope.is_contiguous()
assert kv_buffer.element_size() == 1 and kvrope.dtype == torch.bfloat16
kv = kv_buffer.reshape(-1, ROWB)
kv_rope = kvrope.reshape(-1, ROPE)
assert kv.shape[0] == kv_rope.shape[0]
assert sink.dtype == torch.float32 and sink.is_contiguous() and sink.numel() == H
for t in (qo_indptr, kv_indptr, kv_page_indices):
assert t.dtype == torch.int32 and t.is_contiguous(), t.dtype
assert output.dtype == torch.bfloat16 and output.shape[1:] == (H, DV)
assert output.stride(2) == 1
_check_bounds(q_kv_bounds, q.shape[0])
cfg = _plan(H, msq, B, "bf16", hint, _num_cus(q.device.index), grouped)
if B > 0:
_launch(
cfg,
B,
q,
qrope,
kv,
kv_rope,
qo_indptr,
kv_indptr,
kv_page_indices,
sink,
output,
output.stride(0),
output.stride(1),
bounds=q_kv_bounds,
)
return output
def flydsl_mla_v4_decode_fused(
q,
q_rope,
kv,
kv_rope,
kv_indptr,
kv_indices,
sink,
positions,
freqs,
compress_ratio,
*,
qo_indptr=None,
max_seqlen_q: int = 1,
q_kv_bounds=None,
):
"""DeepSeek-V4 decode tail in one launch: attention + split merge + inverse RoPE of
dims 448..511 + wo_a mxfp8 quant.
q / q_rope / kv / kv_rope / sink: as flydsl_mla_decode_fwd_v4_nm; one sequence per
token (kv_indptr [T + 1]) unless q_kv_bounds + qo_indptr + max_seqlen_q give
grouped-verify streams. positions [T] int64; freqs [max_pos, 64] fp32 =
view_as_real(freqs_cis). compress_ratio: 4 CSA, 128 HCA, else SWA-only.
Returns (xq [T, H*512] fp8_e4m3fn, xs [T, H*4] e8m0); tokens without keys get NaN.
"""
T, H = q.shape[0], q.shape[1]
msq = int(max_seqlen_q)
grouped = q_kv_bounds is not None
if not (msq == 1 or grouped) or not flydsl_mla_v4_decode_supported(H, msq, grouped):
raise RuntimeError(
f"flydsl_mla_v4_decode_fused: no variant for num_heads={H}, "
f"max_seqlen_q={msq}, grouped={grouped}"
)
if grouped:
assert qo_indptr is not None and qo_indptr.dtype == torch.int32
B = qo_indptr.numel() - 1
else:
B = T
assert q.shape[2] == ROWB and q_rope.shape[1:] == (H, ROPE)
assert q.is_contiguous() and q_rope.is_contiguous() and q.element_size() == 1
assert q_rope.dtype == kv_rope.dtype == torch.bfloat16
assert kv_indptr.dtype == kv_indices.dtype == torch.int32
assert sink.dtype == freqs.dtype == torch.float32 and positions.dtype == torch.int64
assert sink.is_contiguous() and freqs.is_contiguous() and kv.element_size() == 1
assert freqs.shape[-1] == ROPE
kv = kv.reshape(-1, ROWB)
kv_rope = kv_rope.reshape(-1, ROPE)
assert kv.shape[0] == kv_rope.shape[0]
xq = torch.empty((T, H * DV), dtype=torch.float8_e4m3fn, device=q.device)
xs = torch.empty((T, H * DV // 128), dtype=torch.uint8, device=q.device)
_check_bounds(q_kv_bounds, T)
hint = {4: "csa", 128: "hca"}.get(compress_ratio, "swa")
cfg = _plan(H, msq, B, "invrope_mxfp8", hint, _num_cus(q.device.index), grouped)
if T > 0 and B > 0:
_launch(
cfg,
B,
q,
q_rope,
kv,
kv_rope,
None if cfg.MSQ == 1 else qo_indptr,
kv_indptr,
kv_indices,
sink,
xq,
0,
0,
xs,
positions,
freqs,
q_kv_bounds,
)
return xq, xs
@@ -0,0 +1,174 @@
# SPDX-License-Identifier: MIT
# Copyright (C) 2024-2026, Advanced Micro Devices, Inc. All rights reserved.
#
# Vendored from ROCm/aiter#5882 (https://github.com/ROCm/aiter/pull/5882),
# commit 03359c43df (03359c43df40abf91ac38a2040c9b8e284d92297),
# file aiter/ops/flydsl/mla_v4_grouped.py.
# Unchanged except for import paths and sglang formatting (isort, ruff format).
# TODO(aiter#5882): remove once ROCm/aiter#5882 is in the aiter pin
"""Grouped DSpark verify streams for the FlyDSL v4 decode.
The num_draft tokens of a request (positions p-nd+1 .. p) share one stream at an HCA /
SWA-only layer: [compressed tail of the last draft (C entries)][SWA ring slots of
positions p-n+1 .. p], n = min(p+1, win+nd-1) (the sglang#41120 layout). q_kv_bounds
gives each draft its exact key set in it: [0, C_q) and its own window.
"""
from __future__ import annotations
import torch
import triton
import triton.language as tl
__all__ = ["flydsl_mla_v4_grouped_verify_meta"]
GROUPED_MAX_MSQ = 8 # drafts per request
@triton.jit
def _grouped_meta_kernel(
idx_ptr,
kv_indptr_ptr,
qo_indptr_ptr,
bnd_ptr,
slot_ptr,
pos_ptr,
tail_len_ptr,
tail_page_ptr,
G,
ring_stride,
swa_pages,
tail_stride,
ND: tl.constexpr,
WIN: tl.constexpr,
TAIL_B: tl.constexpr,
WIN_B: tl.constexpr,
GB: tl.constexpr,
HAS_TAIL: tl.constexpr,
MAXND: tl.constexpr,
):
g = tl.program_id(0)
# start of this group's stream: the previous groups' lengths
base = 0
for off in tl.range(0, G, GB):
gg = off + tl.arange(0, GB)
m = gg < g
last = gg * ND + ND - 1
p = tl.load(pos_ptr + last, mask=m, other=0).to(tl.int32)
if HAS_TAIL:
c = tl.load(tail_len_ptr + last, mask=m, other=0).to(tl.int32)
else:
c = tl.zeros([GB], tl.int32)
n = tl.minimum(p + 1, WIN + ND - 1)
base += tl.sum(tl.where(m, c + n, 0))
first = g * ND
last = first + ND - 1
p = tl.load(pos_ptr + last).to(tl.int32)
if HAS_TAIL:
c = tl.load(tail_len_ptr + last).to(tl.int32)
else:
c = 0
n = tl.minimum(p + 1, WIN + ND - 1)
tl.store(kv_indptr_ptr + g + 1, base + c + n)
tl.store(qo_indptr_ptr + g + 1, first + ND)
if g == 0:
tl.store(kv_indptr_ptr, 0)
tl.store(qo_indptr_ptr, 0)
if HAS_TAIL:
for off in tl.range(0, c, TAIL_B):
jt = off + tl.arange(0, TAIL_B)
mt = jt < c
pi = tl.load(
tail_page_ptr + last.to(tl.int64) * tail_stride + jt, mask=mt, other=-1
)
tl.store(
idx_ptr + base + jt, tl.where(pi >= 0, pi + swa_pages, -1), mask=mt
)
slot = tl.load(slot_ptr + last).to(tl.int64)
i = tl.arange(0, WIN_B)
ap = (p - n + 1 + i).to(tl.int64)
tl.store(
idx_ptr + base + c + i,
(slot * ring_stride + ap % ring_stride).to(tl.int32),
mask=i < n,
)
# per-row bounds, stream relative
q = tl.arange(0, MAXND)
mq = q < ND
t = first + q
pq = tl.load(pos_ptr + t, mask=mq, other=0).to(tl.int32)
if HAS_TAIL:
cq = tl.load(tail_len_ptr + t, mask=mq, other=0).to(tl.int32)
else:
cq = tl.zeros([MAXND], tl.int32)
s0 = p - n + 1
tb = bnd_ptr + t.to(tl.int64) * 4
tl.store(tb + 0, tl.zeros([MAXND], tl.int32), mask=mq)
tl.store(tb + 1, cq, mask=mq)
tl.store(tb + 2, c + tl.maximum(0, pq - WIN + 1 - s0), mask=mq)
tl.store(tb + 3, c + pq - s0 + 1, mask=mq)
def flydsl_mla_v4_grouped_verify_meta(
state_slot: torch.Tensor,
positions: torch.Tensor,
tail_len: torch.Tensor | None,
tail_page_indices: torch.Tensor | None,
*,
win: int,
ring_stride: int,
swa_pages: int,
num_draft: int,
):
"""(kv_indices, kv_indptr, qo_indptr, q_kv_bounds) of the grouped streams, one
graph-capturable launch; N = requests * num_draft tokens, request-major.
state_slot / positions [N]; tail_len [N] / tail_page_indices [N, Wc] int32: the
committed compressed entries per token (None: SWA-only). kv_indices is sized for
the worst case. Use with max_seqlen_q = num_draft.
"""
assert 1 <= num_draft <= GROUPED_MAX_MSQ
dev = positions.device
N = positions.shape[0]
assert N % num_draft == 0, (N, num_draft)
G = N // num_draft
has_tail = tail_len is not None and tail_page_indices is not None
Wc = tail_page_indices.shape[1] if has_tail else 0
idx = torch.empty(
max(G * (Wc + win + num_draft - 1), 1), dtype=torch.int32, device=dev
)
kv_indptr = torch.empty(G + 1, dtype=torch.int32, device=dev)
qo_indptr = torch.empty(G + 1, dtype=torch.int32, device=dev)
bnd = torch.empty((N, 4), dtype=torch.int32, device=dev)
if G == 0:
kv_indptr.zero_()
qo_indptr.zero_()
return idx, kv_indptr, qo_indptr, bnd
assert state_slot.is_contiguous() and positions.is_contiguous()
if has_tail:
assert tail_len.is_contiguous() and tail_page_indices.stride(1) == 1
tp = tail_page_indices if has_tail else positions
_grouped_meta_kernel[(G,)](
idx,
kv_indptr,
qo_indptr,
bnd,
state_slot,
positions,
tail_len if has_tail else positions,
tp,
G,
ring_stride,
swa_pages,
tp.stride(0) if has_tail else 0,
ND=num_draft,
WIN=win,
TAIL_B=min(1024, triton.next_power_of_2(max(Wc, 1))),
WIN_B=triton.next_power_of_2(win + num_draft - 1),
GB=min(1024, triton.next_power_of_2(G)),
HAS_TAIL=has_tail,
MAXND=GROUPED_MAX_MSQ,
num_warps=4,
)
return idx, kv_indptr, qo_indptr, bnd
@@ -27,6 +27,7 @@ on via ``topk_length``); the per-token compressed count is recovered from the
from __future__ import annotations
import logging
from functools import lru_cache
from typing import Optional, Tuple
@@ -49,6 +50,8 @@ from sglang.kernels.ops.attention.dsv4.unified_kv_kernels.paged_prefill import (
sparse_attn_v4_paged_prefill,
)
logger = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# SWA ring scatter
@@ -282,6 +285,289 @@ def _decode_fp8_tail_splits(T: int, compress_ratio: Optional[int]) -> int:
return max(1, min(_DECODE_SPLIT_TAIL_VALUE, _cu_count() // T))
# ---------------------------------------------------------------------------
# Split plan from aiter's v4 nm latency model (ROCm/aiter#6126, ATOM#2462)
# ---------------------------------------------------------------------------
# t(i) ~= L + W(i) * (C + A * kv_len / i) + [i > 1] * (S + D * num_seqs * i),
# W(i) = ceil(num_seqs * ceil(H / 64) * i / CU). The gfx950 fit is aiter's
# (gqa=128, qSeqLen=1, graph replay, uniform KV). Used only when the installed
# aiter predates get_mla_v4_nm_num_kv_splits.
_V4_NM_SPLIT_COST = {"gfx950": (12.74, 3.21, 0.0300, 3.38, 0.076)}
V4_NM_MAX_SPLITS = 16
# rows the constant split table covers; the asm wraps byte offsets past 32768
V4_NM_SPLIT_TABLE_ROWS = 32768
def _ported_num_kv_splits(
num_seqs: int, num_heads: int, kv_len: int, gfx: str, cu_num: int
) -> Optional[int]:
cost = _V4_NM_SPLIT_COST.get(gfx)
if cost is None:
return None
L, C, A, S, D = cost
tg_factor = max(1, -(-num_heads // 64))
kv_len = max(1, kv_len)
best_t, best_s = None, 1
for s in range(1, V4_NM_MAX_SPLITS + 1):
waves = -(-num_seqs * tg_factor * s // cu_num)
t = L + waves * (C + A * kv_len / s)
if s > 1:
t += S + D * num_seqs * s
if best_t is None or t < best_t:
best_t, best_s = t, s
return best_s
@lru_cache(maxsize=None)
def fp8_decode_num_kv_splits(
num_seqs: int, num_heads: int, kv_len: int
) -> Optional[int]:
"""aiter's latency-model split count for ``mla_decode_fwd_v4_nm``, or None
(no model for this arch: keep the existing pick). Host only, cached: the
inputs are fixed per captured graph, so capture and replay agree."""
try:
from aiter.mla import get_mla_v4_nm_num_kv_splits
except ImportError:
get_mla_v4_nm_num_kv_splits = None
if get_mla_v4_nm_num_kv_splits is not None:
return int(get_mla_v4_nm_num_kv_splits(num_seqs, num_heads, kv_len))
from aiter.jit.utils.chip_info import get_gfx
return _ported_num_kv_splits(num_seqs, num_heads, kv_len, get_gfx(), _cu_count())
_split_tables: dict = {}
def fp8_decode_split_table(device: torch.device) -> Optional[torch.Tensor]:
"""``[V4_NM_MAX_SPLITS, V4_NM_SPLIT_TABLE_ROWS + 1]`` int32, row ``s - 1``
the uniform ``split_indptr`` ``[0, s, 2s, ...]``. Built once per device
outside capture and never written again, so every plan is a slice of
constant memory (no per-layer arange). None if it would be built under
capture: that graph keeps the wrapper's own pick."""
device = torch.device(device)
key = device.index if device.index is not None else torch.cuda.current_device()
table = _split_tables.get(key)
if table is None:
if torch.cuda.is_current_stream_capturing():
return None
splits = torch.arange(1, V4_NM_MAX_SPLITS + 1, dtype=torch.int32, device=device)
table = splits[:, None] * torch.arange(
V4_NM_SPLIT_TABLE_ROWS + 1, dtype=torch.int32, device=device
)
_split_tables[key] = table
return table
def fp8_decode_split_plan(
n_seq: int, num_heads: int, kv_len_hint: int, device: torch.device
) -> Optional[Tuple[int, torch.Tensor]]:
"""(num_kv_splits, split_indptr) from the latency model, or None."""
if n_seq > V4_NM_SPLIT_TABLE_ROWS:
return None
s = fp8_decode_num_kv_splits(n_seq, num_heads, int(kv_len_hint))
if s is None:
return None
table = fp8_decode_split_table(device)
if table is None:
return None
return s, table[s - 1, : n_seq + 1]
# ---------------------------------------------------------------------------
# HCA persistent decode (aiter mla_decode_fwd_v4_nm_ps, ROCm/aiter#6126)
# ---------------------------------------------------------------------------
HCA_PS_HEADS = 128
HCA_PS_MAX_ROWS = 32768
HCA_PS_PARTITIONS = 128
_hca_ps_workspaces: dict = {}
def _hca_ps_api():
try:
from aiter.mla import get_mla_v4_nm_ps_workspace, mla_decode_fwd_v4_nm_ps
except ImportError:
return None
return get_mla_v4_nm_ps_workspace, mla_decode_fwd_v4_nm_ps
def _dev_key(device) -> int:
device = torch.device(device)
return device.index if device.index is not None else torch.cuda.current_device()
def prepare_fp8_hca_persist(device, num_heads: int) -> bool:
"""Allocate this device's persistent-decode workspace (~64 MiB) when the
switch is on and the kernel can serve this rank. Call outside capture,
before warmup; idempotent. One workspace per device is enough because
every decode layer runs on the forward's compute stream in order."""
from sglang.srt.environ import envs
from sglang.srt.utils import is_gfx95_supported
if not envs.SGLANG_DSV4_FP8_HCA_PERSIST.get() or num_heads != HCA_PS_HEADS:
return False
if not is_gfx95_supported():
return False
key = _dev_key(device)
if key in _hca_ps_workspaces:
return True
api = _hca_ps_api()
if api is None or torch.cuda.is_current_stream_capturing():
return False
_hca_ps_workspaces[key] = api[0](
torch.device("cuda", key), num_partitions=HCA_PS_PARTITIONS
)
return True
def fp8_hca_persist_wanted(
*,
T: int,
H: int,
v_head_dim: int,
compress_ratio: Optional[int],
max_seqlen_q: int,
per_token: bool,
device,
) -> bool:
from sglang.srt.environ import envs
if not (
compress_ratio == 128
and H == HCA_PS_HEADS
and v_head_dim == 512
and max_seqlen_q == 1
and per_token
and envs.SGLANG_DSV4_FP8_HCA_PERSIST.get()
and envs.SGLANG_DSV4_FP8_HCA_PERSIST_MIN_ROWS.get() <= T <= HCA_PS_MAX_ROWS
):
return False
# eager calls may allocate it late; a graph captured without one keeps asm
return prepare_fp8_hca_persist(device, H)
def decode_fp8_hca_ps(
*,
q: torch.Tensor, # [T, 128, 512] fp8 packed
q_rope: torch.Tensor, # [T, 128, 64] bf16
unified_kv: torch.Tensor,
unified_kv_rope: torch.Tensor,
kv_indices: torch.Tensor,
kv_indptr: torch.Tensor,
attn_sink: torch.Tensor,
) -> torch.Tensor:
"""One-launch HCA decode: the kernel plans the split from ``kv_indptr`` on
the GPU and merges the partials itself. Rows with an empty KV range are
left unwritten (the builders never emit one; see decode_fp8_2buff)."""
T = q.shape[0]
out = q_rope.new_empty((T, HCA_PS_HEADS, 512))
if T == 0:
return out
_hca_ps_api()[1](
q,
q_rope,
unified_kv,
unified_kv_rope,
kv_indptr[: T + 1],
kv_indices,
attn_sink,
_hca_ps_workspaces[_dev_key(q.device)],
out=out,
)
return out
# TODO(aiter#5882): remove once ROCm/aiter#5882 is in the aiter pin
# (with it: the in-tree fallback below, flydsl_v4nm/ and
# SGLANG_DSV4_FLYDSL_DECODE_SOURCE; aiter's copy is then the only one).
_FLYDSL_V4_DECODE_SOURCES = ("auto", "aiter", "intree")
@lru_cache(maxsize=1)
def _flydsl_v4_decode():
"""The FlyDSL v4 nm decode API (ROCm/aiter#5882) or None (asm kernel).
SGLANG_DSV4_FLYDSL_DECODE=0 forces None. SGLANG_DSV4_FLYDSL_DECODE_SOURCE:
auto (default) takes aiter.ops.flydsl when the installed aiter ships it, else
sglang's in-tree copy (flydsl_v4nm); aiter / intree take only that one.
"""
from sglang.srt.environ import envs
if not envs.SGLANG_DSV4_FLYDSL_DECODE.get():
return None
source = envs.SGLANG_DSV4_FLYDSL_DECODE_SOURCE.get()
if source not in _FLYDSL_V4_DECODE_SOURCES:
raise ValueError(
f"SGLANG_DSV4_FLYDSL_DECODE_SOURCE={source!r}, expected one of "
f"{_FLYDSL_V4_DECODE_SOURCES}"
)
fly = None
if source != "intree":
try:
from aiter.ops import flydsl as fly
fly.flydsl_mla_v4_decode_supported # resolves the lazy import
origin = "aiter.ops.flydsl"
except (ImportError, AttributeError):
fly = None
if fly is None and source != "aiter":
try:
from sglang.kernels.ops.attention.dsv4 import flydsl_v4nm as fly
origin = "sglang in-tree copy of ROCm/aiter#5882"
except ImportError as e: # no flydsl, or an aiter before ROCm/aiter#6377
fly = None
logger.warning("in-tree FlyDSL v4 nm decode unavailable: %s", e)
if fly is None:
logger.info("DSv4 fp8 decode: no FlyDSL v4 nm decode (source=%s)", source)
else:
logger.info("DSv4 fp8 decode: FlyDSL v4 nm decode from %s", origin)
return fly
def flydsl_decode_supported(
num_heads: int, max_seqlen_q: int = 1, grouped: bool = False
) -> bool:
fly = _flydsl_v4_decode()
if fly is None:
return False
if num_heads == 128:
from sglang.srt.environ import envs
if not envs.SGLANG_DSV4_FLYDSL_DECODE_H128.get():
return False
return fly.flydsl_mla_v4_decode_supported(num_heads, max_seqlen_q, grouped)
def _stream_hint(compress_ratio: Optional[int]) -> Optional[str]:
# FlyDSL split plans are keyed by stream kind
if compress_ratio is None:
return None
return {0: "swa", 4: "csa", 128: "hca"}[compress_ratio]
def flydsl_grouped_verify_pays(
num_heads: int, num_draft: int, num_reqs: int, compress_ratio: int, epi: str
) -> bool:
"""aiter's measured policy: whether target verify should read one shared
stream per request on this layer kind, for output mode ``epi``."""
return flydsl_decode_supported(
num_heads, num_draft, grouped=True
) and _flydsl_v4_decode().flydsl_mla_v4_grouped_pays(
num_heads, num_draft, num_reqs, hint=_stream_hint(compress_ratio), epi=epi
)
def build_grouped_verify_streams(
state_slot: torch.Tensor, positions: torch.Tensor, *args, **kwargs
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""(kv_indices, kv_indptr, qo_indptr, q_kv_bounds): one stream per request,
with per-draft key ranges equal to each draft's per-token stream."""
return _flydsl_v4_decode().flydsl_mla_v4_grouped_verify_meta(
state_slot.contiguous(), positions.contiguous(), *args, **kwargs
)
def decode_fp8_2buff(
*,
q: torch.Tensor, # [T, H, nope_row_bytes] fp8 packed nope + inline e8m0 scale
@@ -296,8 +582,17 @@ def decode_fp8_2buff(
max_seqlen_q: int = 1,
num_kv_splits: Optional[int] = None,
compress_ratio: Optional[int] = None,
q_kv_bounds: Optional[torch.Tensor] = None,
kv_len_hint: Optional[int] = None,
) -> torch.Tensor:
"""Decode over the two-pool fp8 unified_kv, through aiter's v4 nm asm kernel.
"""Decode over the two-pool fp8 unified_kv, through aiter's FlyDSL v4 nm
decode (split merge in the same launch) where available, else the asm kernel.
On the asm kernel, with ``SGLANG_DSV4_FP8_HCA_PERSIST`` the HCA stream
(H128, one q row per sequence) takes the persistent kernel, and with
``SGLANG_DSV4_FP8_DECODE_SPLIT_MODEL`` plus ``kv_len_hint`` (the longest
KV a row of this stream can see, fixed per server) the split count comes
from aiter's latency model with a constant ``split_indptr``.
Q arrives in the same packed form as the pool rows (nope fp8 + duplicated
e8m0 tile scales) with its rope half beside it in bf16, which is why this
@@ -314,6 +609,9 @@ def decode_fp8_2buff(
``max_seqlen_q`` above 1 puts several q rows in one tile, which the v4
dispatcher serves for gqa=16 at 1, 2 and 4 off the same 64-row tile.
``q_kv_bounds`` (FlyDSL only): the grouped verify streams of
``build_grouped_verify_streams``.
"""
from aiter.mla import mla_decode_fwd_v4_nm
@@ -350,15 +648,52 @@ def decode_fp8_2buff(
rows = unified_kv.shape[0]
out = q_rope.new_empty((n_seq * max_seqlen_q, H, v_head_dim))
# Left None, the wrapper's occupancy heuristic picks it, folds the cross-split
# merge back into `out`, and leaves the final bf16 there whether or not it
# split. Pinning it to 1 costs 6.9x at bs=1 kv=2048.
# Sequences, not tokens: the wave-fitting rule counts the workgroups the
# kernel launches, which is n_seq x splits. They are the same number on the
# per-token path and differ once q rows share a tile.
if num_kv_splits is None and n_seq > _DECODE_SPLIT_TAIL_MIN_TOKENS:
num_kv_splits = _decode_fp8_tail_splits(n_seq, compress_ratio)
mla_decode_fwd_v4_nm(
# FlyDSL is a drop-in for the asm kernel: its own capture-safe split plan,
# split merge in the same launch
if flydsl_decode_supported(H, max_seqlen_q, q_kv_bounds is not None):
decode_fwd = _flydsl_v4_decode().flydsl_mla_decode_fwd_v4_nm
kwargs = dict(hint=_stream_hint(compress_ratio), q_kv_bounds=q_kv_bounds)
else:
assert q_kv_bounds is None, "grouped verify streams need FlyDSL"
if fp8_hca_persist_wanted(
T=T,
H=H,
v_head_dim=v_head_dim,
compress_ratio=compress_ratio,
max_seqlen_q=max_seqlen_q,
per_token=n_seq == T,
device=q.device,
):
return decode_fp8_hca_ps(
q=q,
q_rope=q_rope,
unified_kv=unified_kv,
unified_kv_rope=unified_kv_rope,
kv_indices=kv_indices,
kv_indptr=kv_indptr,
attn_sink=attn_sink,
)
split_indptr = None
if num_kv_splits is None and kv_len_hint is not None:
from sglang.srt.environ import envs
if envs.SGLANG_DSV4_FP8_DECODE_SPLIT_MODEL.get():
plan = fp8_decode_split_plan(n_seq, H, kv_len_hint, q.device)
if plan is not None:
num_kv_splits, split_indptr = plan
# Left None, the wrapper's occupancy heuristic picks it, folds the
# cross-split merge back into `out`, and leaves the final bf16 there
# whether or not it split. Pinning it to 1 costs 6.9x at bs=1 kv=2048.
# Sequences, not tokens: the wave-fitting rule counts the workgroups the
# kernel launches, which is n_seq x splits. They are the same number on
# the per-token path and differ once q rows share a tile.
if num_kv_splits is None and n_seq > _DECODE_SPLIT_TAIL_MIN_TOKENS:
num_kv_splits = _decode_fp8_tail_splits(n_seq, compress_ratio)
decode_fwd, kwargs = (
mla_decode_fwd_v4_nm,
dict(num_kv_splits=num_kv_splits, split_indptr=split_indptr),
)
decode_fwd(
q,
q_rope,
unified_kv.view(rows, 1, 1, row_bytes),
@@ -369,7 +704,7 @@ def decode_fp8_2buff(
kv_indices,
max_seqlen_q,
sink=attn_sink,
num_kv_splits=num_kv_splits,
**kwargs,
)
# No empty-segment mask: a CG-padded row gets seq_len 1 on the ring slot
# ReqToTokenPool reserves, so the builders can't emit a zero-length one, and
@@ -378,6 +713,41 @@ def decode_fp8_2buff(
return out[:T]
def decode_fp8_2buff_wo_a_mxfp8(
*,
q: torch.Tensor, # [T, H, nope_row_bytes] fp8 packed nope + inline e8m0 scale
q_rope: torch.Tensor, # [T, H, rope_dim] bf16
unified_kv: torch.Tensor, # [rows, nope_row_bytes] fp8
unified_kv_rope: torch.Tensor, # [rows, rope_dim] bf16
kv_indices: torch.Tensor,
kv_indptr: torch.Tensor,
attn_sink: torch.Tensor, # [H] fp32
positions: torch.Tensor, # [T] int64
rope_freqs: torch.Tensor, # [max_pos, rope_dim] fp32, view_as_real(freqs_cis)
compress_ratio: int,
qo_indptr: Optional[torch.Tensor] = None,
max_seqlen_q: int = 1,
q_kv_bounds: Optional[torch.Tensor] = None,
) -> Tuple[torch.Tensor, torch.Tensor]:
"""``decode_fp8_2buff`` + inverse RoPE + wo_a mxfp8 quant in one FlyDSL
launch. Returns the wo_a operands (fp8 [T, H*512], e8m0 [T, H*4])."""
return _flydsl_v4_decode().flydsl_mla_v4_decode_fused(
q,
q_rope,
unified_kv,
unified_kv_rope,
kv_indptr,
kv_indices,
attn_sink,
positions,
rope_freqs,
compress_ratio,
qo_indptr=qo_indptr,
max_seqlen_q=max_seqlen_q,
q_kv_bounds=q_kv_bounds,
)
@triton.jit
def _fill_compress_tail_kernel(
indices_ptr, # [*] int32 (out)
+25
View File
@@ -1613,6 +1613,31 @@ class Envs:
# only way to ask; on separate-KV it is the reverse -- --kv-cache-dtype
# picks the buffer dtype and this switch is inert.
SGLANG_DSV4_UNIFIED_KV_FP8 = EnvBool(False)
# fp8 unified_kv decode/verify on aiter's FlyDSL v4 nm decode when the
# installed aiter ships it; 0 keeps the asm kernel.
SGLANG_DSV4_FLYDSL_DECODE = EnvBool(True)
# ... which copy: "auto" = aiter's when the installed aiter ships it, else
# sglang's in-tree copy; "aiter" = aiter's only (asm without it); "intree" =
# the in-tree copy only.
# TODO(aiter#5882): remove once ROCm/aiter#5882 is in the aiter pin
SGLANG_DSV4_FLYDSL_DECODE_SOURCE = EnvStr("auto")
# ... also with 128 local heads (DP attention). Off: there the asm path
# below (split plan + HCA persistent, ROCm/aiter#6126) measured the same
# DP8 DSpark decode step on MI355X without depending on FlyDSL.
SGLANG_DSV4_FLYDSL_DECODE_H128 = EnvBool(False)
# fp8 unified_kv asm decode (when FlyDSL is off or absent): pick the KV
# split count from aiter's v4 nm latency model (ROCm/aiter#6126; ported
# when aiter predates it) with a constant split_indptr, per stream kind.
SGLANG_DSV4_FP8_DECODE_SPLIT_MODEL = EnvBool(True)
# context the HCA split plan is sized for, capped at the server context
# length (0: the server context length). Sizing for 1M over-splits HCA at
# AgentX lengths (32k-200k).
SGLANG_DSV4_FP8_DECODE_SPLIT_CTX = EnvInt(200_000)
# fp8 unified_kv asm decode: HCA layers with 128 local heads (DP attention)
# take aiter's persistent mla_decode_fwd_v4_nm_ps (in-kernel split + merge)
# for MIN_ROWS..32768 rows; needs aiter >= ROCm/aiter#6126.
SGLANG_DSV4_FP8_HCA_PERSIST = EnvBool(True)
SGLANG_DSV4_FP8_HCA_PERSIST_MIN_ROWS = EnvInt(15)
# DeepSeek-V4.1 engram host table: keep the tables in host memory (layout
# below) and gather rows from the GPU instead of sharding them over HBM.
@@ -270,6 +270,16 @@ class UnifiedKvMetadata:
gasm_kv_indptr: Optional[torch.Tensor] = None
gasm_qo_indptr: Optional[torch.Tensor] = None
# Grouped target-verify streams for the FlyDSL fp8 decode: one per request
# on HCA / SWA-only layers, with per-draft key ranges (q_kv_bounds).
grp_qo_indptr: Optional[torch.Tensor] = None
grp_hca_indices: Optional[torch.Tensor] = None
grp_hca_indptr: Optional[torch.Tensor] = None
grp_hca_bounds: Optional[torch.Tensor] = None
grp_swa_indices: Optional[torch.Tensor] = None
grp_swa_indptr: Optional[torch.Tensor] = None
grp_swa_bounds: Optional[torch.Tensor] = None
# prefill/extend per-token mapping
pf_state_slot: Optional[torch.Tensor] = None
pf_chunk_start: Optional[torch.Tensor] = None
@@ -298,6 +308,13 @@ class UnifiedKvMetadata:
"gasm_indices",
"gasm_kv_indptr",
"gasm_qo_indptr",
"grp_qo_indptr",
"grp_hca_indices",
"grp_hca_indptr",
"grp_hca_bounds",
"grp_swa_indices",
"grp_swa_indptr",
"grp_swa_bounds",
"csa_indices",
"csa_indptr",
"pf_state_slot",
@@ -330,6 +347,13 @@ class UnifiedKvMetadata:
"gasm_indices",
"gasm_kv_indptr",
"gasm_qo_indptr",
"grp_qo_indptr",
"grp_hca_indices",
"grp_hca_indptr",
"grp_hca_bounds",
"grp_swa_indices",
"grp_swa_indptr",
"grp_swa_bounds",
"csa_indices",
"csa_indptr",
"pf_state_slot",
@@ -971,6 +995,17 @@ class DeepseekV4HipRadixBackend(
# DSpark draft workers verify gamma rows; the server arg keeps the
# CUDA-side convention gamma + 1.
self.target_verify_num_draft_tokens = self.speculative_num_draft_tokens - 1
self._fp8_decode_num_heads = (
model_runner.model_config.num_attention_heads // get_parallel().attn_tp_size
)
# output mode of the fp8 verify: deepseek_v4 fuses the fp8 wo_a quant into it
self._fp8_verify_epi = (
"invrope_mxfp8"
if any(getattr(m, "wo_a_fp8", False) for m in model_runner.model.modules())
else "bf16"
)
self._fp8_split_kv_hint: Dict[int, int] = {}
self._init_fp8_asm_decode_plan()
# Past MAX_FUSED_ROWS the fp4 schedule falls back to AITER's preamble,
# which frees the scratch its kernels read -- not capture-safe.
self._fp4_graph_row_limit: Optional[int] = None
@@ -2409,6 +2444,37 @@ class DeepseekV4HipRadixBackend(
if current_raw is not None:
self.forward_metadata = current_raw
def _init_fp8_asm_decode_plan(self) -> None:
"""fp8 unified_kv asm decode: per-stream KV bound for the split plan
and, at init (outside capture, before warmup), the constant split
table and the HCA persistent workspace."""
from sglang.kernels.ops.attention.dsv4.unified_kv_kernels.env_gate import (
is_unified_kv_fp8,
)
if self.is_draft_worker or not is_unified_kv_fp8():
return
from sglang.kernels.ops.attention.dsv4.unified_kv_kernels import runtime
win = self.token_to_kv_pool.unified_swa_window
ctx = min(
envs.SGLANG_DSV4_FP8_DECODE_SPLIT_CTX.get() or self.max_context_len,
self.max_context_len,
)
# longest KV one decode row of each stream can see
self._fp8_split_kv_hint = {
0: win,
4: win + self.index_topk,
128: win + ctx // 128,
}
if envs.SGLANG_DSV4_FP8_DECODE_SPLIT_MODEL.get():
runtime.fp8_decode_split_table(self.device)
if runtime.prepare_fp8_hca_persist(self.device, self._fp8_decode_num_heads):
logger.info(
"DSV4 fp8 decode: HCA layers on the persistent asm kernel "
f"(rows >= {envs.SGLANG_DSV4_FP8_HCA_PERSIST_MIN_ROWS.get()})"
)
def _attach_unified_kv_decode_streams(
self,
core: DSV4AttnMetadata,
@@ -2418,6 +2484,7 @@ class DeepseekV4HipRadixBackend(
# state_slot maps each query token to its request slot;
# target-verify repeats request slots for the draft tokens.
from sglang.kernels.ops.attention.dsv4.unified_kv_kernels.env_gate import (
is_unified_kv_fp8,
is_unified_kv_triton,
)
@@ -2493,6 +2560,43 @@ class DeepseekV4HipRadixBackend(
block_q=_GROUPED_ASM_BLOCK_Q,
)
u = core.unified
u.grp_qo_indptr = None
u.grp_hca_indices = u.grp_hca_indptr = u.grp_hca_bounds = None
u.grp_swa_indices = u.grp_swa_indptr = u.grp_swa_bounds = None
num_reqs = N // num_draft if num_draft > 1 and N % num_draft == 0 else 0
if self.is_dspark and num_reqs and is_unified_kv_fp8():
grouped = functools.partial(
runtime.build_grouped_verify_streams,
state_slot,
core.positions_casual,
num_draft=num_draft,
win=pool.unified_swa_window,
ring_stride=pool.unified_swa_ring_size,
swa_pages=pool.unified_swa_pages,
)
pays = functools.partial(
runtime.flydsl_grouped_verify_pays,
self._fp8_decode_num_heads,
num_draft,
num_reqs,
epi=self._fp8_verify_epi,
)
if pays(128):
(
u.grp_hca_indices,
u.grp_hca_indptr,
u.grp_qo_indptr,
u.grp_hca_bounds,
) = grouped(hca_len, hca_page_indices)
if pays(0):
(
u.grp_swa_indices,
u.grp_swa_indptr,
u.grp_qo_indptr,
u.grp_swa_bounds,
) = grouped(None, None)
# SWA ring write target, same value for every layer this forward.
req_slot = state_slot.to(torch.int64)
core.unified.swa_loc = (
@@ -2574,9 +2678,12 @@ class DeepseekV4HipRadixBackend(
save_kv_cache: bool = True,
q_rope: Optional[torch.Tensor] = None,
k_rope: Optional[torch.Tensor] = None,
) -> torch.Tensor:
wo_a_quant_freqs: Optional[torch.Tensor] = None,
) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]:
"""q_rope present: packed fp8 q over the two-pool fp8 layout (asm decode;
prefill also needs k_rope). Absent: plain bf16 q and pool (Triton)."""
prefill also needs k_rope). Absent: plain bf16 q and pool (Triton).
wo_a_quant_freqs (fp8 decode): return the inverse-RoPE'd wo_a mxfp8
operands instead of the attention output."""
from sglang.kernels.ops.attention.dsv4.unified_kv_kernels import runtime
pool = self.token_to_kv_pool
@@ -2650,9 +2757,44 @@ class DeepseekV4HipRadixBackend(
"the v4 nm asm kernel hardcodes 1/sqrt(512), this backend is "
f"at {self.softmax_scale}"
)
qo_indptr, max_seqlen_q, q_kv_bounds = None, 1, None
grp = {
128: (
unified_metadata.grp_hca_indices,
unified_metadata.grp_hca_indptr,
unified_metadata.grp_hca_bounds,
),
0: (
unified_metadata.grp_swa_indices,
unified_metadata.grp_swa_indptr,
unified_metadata.grp_swa_bounds,
),
}.get(compress_ratio)
if verify_as_decode and grp is not None and grp[0] is not None:
kv_indices, kv_indptr, q_kv_bounds = grp
qo_indptr = unified_metadata.grp_qo_indptr
max_seqlen_q = self.target_verify_num_draft_tokens
if wo_a_quant_freqs is not None:
return runtime.decode_fp8_2buff_wo_a_mxfp8(
q=q,
q_rope=q_rope,
unified_kv=unified,
unified_kv_rope=pool.get_unified_kv_rope(layer_id),
kv_indices=kv_indices,
kv_indptr=kv_indptr,
attn_sink=attn_sink,
positions=positions,
rope_freqs=wo_a_quant_freqs,
compress_ratio=compress_ratio,
qo_indptr=qo_indptr,
max_seqlen_q=max_seqlen_q,
q_kv_bounds=q_kv_bounds,
)
gasm = (
unified_metadata.gasm_indices
if compress_ratio == 128 and q.shape[1] == _GROUPED_ASM_GQA
if compress_ratio == 128
and q.shape[1] == _GROUPED_ASM_GQA
and q_kv_bounds is None
else None
)
if gasm is not None:
@@ -2679,6 +2821,14 @@ class DeepseekV4HipRadixBackend(
attn_sink=attn_sink,
v_head_dim=layer.v_head_dim,
compress_ratio=compress_ratio,
qo_indptr=qo_indptr,
max_seqlen_q=max_seqlen_q,
q_kv_bounds=q_kv_bounds,
kv_len_hint=(
self._fp8_split_kv_hint.get(compress_ratio)
if qo_indptr is None
else None
),
)
from sglang.kernels.ops.attention.dsv4.unified_kv_kernels.paged_decode import (
_kv_splits_for_stream,
@@ -3034,8 +3184,9 @@ class DeepseekV4HipRadixBackend(
q_rope: Optional[torch.Tensor] = None,
k_rope: Optional[torch.Tensor] = None,
inv_rope: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
wo_a_quant_freqs: Optional[torch.Tensor] = None,
**_,
) -> torch.Tensor:
) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]:
"""inv_rope=(freqs_real, positions) has the kernel apply the inverse RoPE
to the last freqs_real.shape[-1] dims of every output head."""
if self.mtp_enabled and forward_batch.forward_mode.is_idle():
@@ -3066,6 +3217,7 @@ class DeepseekV4HipRadixBackend(
save_kv_cache=save_kv_cache,
q_rope=q_rope,
k_rope=k_rope,
wo_a_quant_freqs=wo_a_quant_freqs,
)
assert inv_rope is None, (
"the unified_kv model path applies its inverse RoPE"
@@ -230,6 +230,24 @@ def apply_wo_a_fp8_mxscale_fused_invrope(
)
def apply_wo_a_fp8_mxscale_prequant(
o_fp8: torch.Tensor,
o_scale: torch.Tensor,
weight: torch.Tensor,
weight_scale: torch.Tensor,
) -> torch.Tensor:
"""fp8 ``wo_a`` on ``quant_wo_a_act_mxfp8``-layout operands flattened per
token (fp8 ``[T, G*D]``, e8m0 ``[T, G*D/128]``). Returns bf16 ``[T, G, R]``."""
T, G = o_fp8.shape[0], weight.shape[0]
return _batched_gemm_a8w8_mxscale(
o_fp8.view(T, G, -1),
weight,
o_scale.view(T, G, -1),
weight_scale,
dtype=torch.bfloat16,
)
def _is_power_of_two(scale: torch.Tensor) -> bool:
"""True when every fp32 scale is an exact power of two (zero mantissa)."""
bits = scale.detach().float().contiguous().view(torch.int32)
@@ -287,6 +305,7 @@ def wo_a_weight_scale_to_e8m0(
__all__ = [
"WO_A_MXFP8_GROUP_SIZE",
"apply_wo_a_fp8_mxscale",
"apply_wo_a_fp8_mxscale_prequant",
"is_wo_a_fp8_mxscale_supported",
"quant_wo_a_act_mxfp8",
"wo_a_weight_scale_to_e8m0",
+37 -1
View File
@@ -418,6 +418,7 @@ _wo_a_aiter_batched_gemm_disabled = False
# ``None`` here means the platform keeps the bf16 absorb GEMM.
_wo_a_fp8_mxscale = None
_wo_a_fp8_mxscale_fused_invrope = None
_wo_a_fp8_mxscale_prequant = None
_wo_a_weight_scale_to_e8m0 = None
_hip = None
if _is_hip:
@@ -425,6 +426,7 @@ if _is_hip:
from sglang.srt.models.deepseek_common.amd.deepseek_v4_wo_a_fp8 import (
apply_wo_a_fp8_mxscale,
apply_wo_a_fp8_mxscale_fused_invrope,
apply_wo_a_fp8_mxscale_prequant,
is_wo_a_fp8_fused_invrope_supported,
is_wo_a_fp8_mxscale_supported,
wo_a_weight_scale_to_e8m0,
@@ -440,6 +442,7 @@ if _is_hip:
and is_wo_a_fp8_fused_invrope_supported()
):
_wo_a_fp8_mxscale_fused_invrope = apply_wo_a_fp8_mxscale_fused_invrope
_wo_a_fp8_mxscale_prequant = apply_wo_a_fp8_mxscale_prequant
@functools.lru_cache(maxsize=1)
@@ -1077,6 +1080,27 @@ class MqaAttentionBase(nn.Module):
# Other FlashMLA implementations retain their existing padded shape.
return 64 if self.n_local_heads <= 64 else self.n_heads
def _wo_a_quant_freqs(
self, forward_batch: ForwardBatch, q_rope: Optional[torch.Tensor]
) -> Optional[torch.Tensor]:
"""Rope table when the fp8 unified_kv decode (aiter FlyDSL) should also
do the inverse RoPE and the wo_a mxfp8 quant; else None."""
mode = forward_batch.forward_mode
if q_rope is None or not (mode.is_decode() or mode.is_target_verify()):
return None
if not hasattr(self, "_wo_a_freqs"):
from sglang.kernels.ops.attention.dsv4.unified_kv_kernels import runtime
fused = (
_wo_a_fp8_mxscale_prequant is not None
and self.wo_a_fp8
and runtime.flydsl_decode_supported(q_rope.shape[1])
)
self._wo_a_freqs = (
torch.view_as_real(self.freqs_cis).flatten(1) if fused else None
)
return self._wo_a_freqs
def _local_attn_sink(self, kernel_num_heads: Optional[int] = None) -> torch.Tensor:
if self.attn_tp_size == 1:
return self.attn_sink
@@ -2559,6 +2583,7 @@ class MQALayer(MqaAttentionBase):
if _is_hip
else None
)
wo_a_quant_freqs = None
if unified:
# only the HIP radix backend takes these; passing them always would
# leave non-ROCm depending on the **_ in its forward() to drop them, and
@@ -2570,6 +2595,9 @@ class MQALayer(MqaAttentionBase):
rope_kwargs["k_rope"] = k_rope
if inv_rope is not None:
rope_kwargs["inv_rope"] = inv_rope
wo_a_quant_freqs = self._wo_a_quant_freqs(forward_batch, q_rope)
if wo_a_quant_freqs is not None:
rope_kwargs["wo_a_quant_freqs"] = wo_a_quant_freqs
o = attn_backend.forward(
q=q_out if q_out is not None else q,
k=attn_k,
@@ -2613,7 +2641,15 @@ class MQALayer(MqaAttentionBase):
**({} if inv_rope is None else {"inv_rope": inv_rope}),
)
o = o[:, tp_slice, :]
if (
if wo_a_quant_freqs is not None:
# the decode already returned the wo_a mxfp8 (codes, scales)
G = self.n_local_groups
o = _wo_a_fp8_mxscale_prequant(
*o,
self.wo_a.weight.view(G, self.o_lora_rank, -1),
self.wo_a.weight_scale_inv.data,
)
elif (
self.wo_a_fp8
and _wo_a_fp8_mxscale_fused_invrope is not None
and not _is_npu