[Perf] [AMD] dsv4: fp8 unified_kv decode/verify on FlyDSL v4 nm decode with a latency-model split plan (1/8) (#44016)
Co-authored-by: Duyi-Wang <duyi.wang@amd.com>
This commit is contained in:
@@ -277,6 +277,19 @@ for _mod, _fn, _backend, _device in [
|
||||
KernelBackend.FLYDSL,
|
||||
CapabilityRequirement.HIP,
|
||||
),
|
||||
# TODO(aiter#5882): remove once ROCm/aiter#5882 is in the aiter pin
|
||||
(
|
||||
"dsv4.flydsl_v4nm.mla_v4_decode",
|
||||
"flydsl_mla_decode_fwd_v4_nm",
|
||||
KernelBackend.FLYDSL,
|
||||
CapabilityRequirement.HIP,
|
||||
),
|
||||
(
|
||||
"dsv4.flydsl_v4nm.mla_v4_decode",
|
||||
"flydsl_mla_v4_decode_fused",
|
||||
KernelBackend.FLYDSL,
|
||||
CapabilityRequirement.HIP,
|
||||
),
|
||||
]:
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
# TODO(aiter#5882): remove once ROCm/aiter#5882 is in the aiter pin
|
||||
"""In-tree copy of ROCm/aiter#5882: DeepSeek-V4 fp8 MLA decode (v4 nm contract)
|
||||
in one FlyDSL launch on gfx950, for aiter pins that do not ship it yet.
|
||||
|
||||
Vendored from ROCm/aiter#5882 at commit 03359c43df (the head adapted to
|
||||
ROCm/aiter#6377); see the per-file headers. The entry points match the ones
|
||||
the PR adds to ``aiter.ops.flydsl``. ``unified_kv_kernels.runtime`` prefers
|
||||
aiter's own copy and only falls back to this one (SGLANG_DSV4_FLYDSL_DECODE_SOURCE).
|
||||
|
||||
Needs an aiter with ROCm/aiter#6377 (typed ``_to_ptr_global``): importing this
|
||||
package raises ImportError on an older aiter, so callers fall back to asm.
|
||||
"""
|
||||
|
||||
import inspect
|
||||
|
||||
# AITER owns the FlyDSL toolchain bootstrap and the shared tensor/buffer shims.
|
||||
# Import it before the local kernel modules so its FlyDSL setup is active.
|
||||
import aiter as _aiter # noqa: F401
|
||||
from aiter.ops.flydsl.kernels import communication_ops_utils as _cu
|
||||
|
||||
if "dtype" not in inspect.signature(_cu._to_ptr_global).parameters:
|
||||
raise ImportError(
|
||||
"the in-tree FlyDSL v4 nm decode (ROCm/aiter#5882) needs an aiter with "
|
||||
"ROCm/aiter#6377 (typed communication_ops_utils._to_ptr_global)"
|
||||
)
|
||||
|
||||
from .mla_v4_decode import ( # noqa: E402
|
||||
flydsl_mla_decode_fwd_v4_nm,
|
||||
flydsl_mla_v4_decode_fused,
|
||||
flydsl_mla_v4_decode_supported,
|
||||
flydsl_mla_v4_grouped_pays,
|
||||
)
|
||||
from .mla_v4_grouped import flydsl_mla_v4_grouped_verify_meta # noqa: E402
|
||||
|
||||
__all__ = [
|
||||
"flydsl_mla_decode_fwd_v4_nm",
|
||||
"flydsl_mla_v4_decode_fused",
|
||||
"flydsl_mla_v4_decode_supported",
|
||||
"flydsl_mla_v4_grouped_pays",
|
||||
"flydsl_mla_v4_grouped_verify_meta",
|
||||
]
|
||||
@@ -0,0 +1,2 @@
|
||||
# TODO(aiter#5882): remove once ROCm/aiter#5882 is in the aiter pin
|
||||
"""FlyDSL kernel bodies of the in-tree ROCm/aiter#5882 copy (see the package)."""
|
||||
@@ -0,0 +1,252 @@
|
||||
# SPDX-License-Identifier: MIT
|
||||
# Copyright (C) 2024-2026, Advanced Micro Devices, Inc. All rights reserved.
|
||||
#
|
||||
# Vendored from ROCm/aiter#5882 (https://github.com/ROCm/aiter/pull/5882),
|
||||
# commit 03359c43df (03359c43df40abf91ac38a2040c9b8e284d92297),
|
||||
# file aiter/ops/flydsl/kernels/mla_v4_decode_common.py.
|
||||
# Unchanged except for import paths and sglang formatting (isort, ruff format).
|
||||
# TODO(aiter#5882): remove once ROCm/aiter#5882 is in the aiter pin
|
||||
|
||||
"""Shared helpers of the FlyDSL v4 nm MLA decode (mla_v4_decode_gfx950.py): math
|
||||
intrinsics, cross-lane butterflies, the global atomics / loads / fences of the
|
||||
cross-split merge protocol, and the mxfp8 (ceil-e8m0) block scale."""
|
||||
|
||||
import flydsl.compiler as flyc
|
||||
import flydsl.expr as fx
|
||||
from aiter.ops.flydsl.kernels import communication_ops_utils as cu
|
||||
from flydsl._mlir.dialects import llvm as _llvm_d
|
||||
from flydsl.expr import arith as _arith
|
||||
from flydsl.expr.typing import T
|
||||
|
||||
FP8_MAX = 448.0
|
||||
EX_FLOOR = 94 # smallest e8m0 block scale emitted: 2^(94 - 127)
|
||||
|
||||
|
||||
def exp2(x):
|
||||
return fx.Float32(fx.rocdl.exp2(T.f32, fx.Float32(x).ir_value()))
|
||||
|
||||
|
||||
def rcp(x):
|
||||
return fx.Float32(fx.rocdl.rcp(T.f32, fx.Float32(x).ir_value()))
|
||||
|
||||
|
||||
def ldexp(x, e):
|
||||
return fx.Float32(
|
||||
_llvm_d.call_intrinsic(
|
||||
T.f32,
|
||||
"llvm.ldexp.f32.i32",
|
||||
[fx.Float32(x).ir_value(), fx.Int32(e).ir_value()],
|
||||
[],
|
||||
[],
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def fabs_f32(x):
|
||||
return fx.Float32(
|
||||
_llvm_d.call_intrinsic(
|
||||
T.f32, "llvm.fabs.f32", [fx.Float32(x).ir_value()], [], []
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def maxnum_f32(a, b):
|
||||
return fx.Float32(
|
||||
_llvm_d.call_intrinsic(
|
||||
T.f32,
|
||||
"llvm.maxnum.f32",
|
||||
[fx.Float32(a).ir_value(), fx.Float32(b).ir_value()],
|
||||
[],
|
||||
[],
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def fabs_bits(x):
|
||||
return fabs_f32(x).bitcast(fx.Int32)
|
||||
|
||||
|
||||
def udiv(a, b):
|
||||
return fx.Int32(_arith.divui(fx.Int32(a).ir_value(), fx.Int32(b).ir_value()))
|
||||
|
||||
|
||||
def urem(a, b):
|
||||
return fx.Int32(_arith.remui(fx.Int32(a).ir_value(), fx.Int32(b).ir_value()))
|
||||
|
||||
|
||||
def smax_i32(a, b):
|
||||
return (a > b).select(a, b)
|
||||
|
||||
|
||||
def smin_i32(a, b):
|
||||
return (a < b).select(a, b)
|
||||
|
||||
|
||||
def dpp_i32(x, ctrl):
|
||||
return fx.Int32(
|
||||
fx.rocdl.update_dpp(
|
||||
T.i32, fx.Int32(0).ir_value(), fx.Int32(x).ir_value(), ctrl, 0xF, 0xF, True
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def plswap(x, width, op2):
|
||||
st = _llvm_d.StructType.get_literal([T.i32, T.i32])
|
||||
op = fx.rocdl.permlane16_swap if width == 16 else fx.rocdl.permlane32_swap
|
||||
r = op(st, fx.Int32(x).ir_value(), fx.Int32(x).ir_value(), False, False)
|
||||
a = fx.Int32(_llvm_d.extractvalue(T.i32, r, [0]))
|
||||
b = fx.Int32(_llvm_d.extractvalue(T.i32, r, [1]))
|
||||
return op2(a, b)
|
||||
|
||||
|
||||
_DPP_XOR = {1: 0xB1, 2: 0x4E, 4: 0x141, 8: 0x128}
|
||||
|
||||
|
||||
def butterfly_i32(x, nlanes, op2, first=1):
|
||||
k = first
|
||||
while k < nlanes * first:
|
||||
if k <= 8:
|
||||
x = op2(x, dpp_i32(x, _DPP_XOR[k]))
|
||||
else:
|
||||
x = plswap(x, k, op2)
|
||||
k *= 2
|
||||
return x
|
||||
|
||||
|
||||
def mxfp8_exp(amb):
|
||||
# e8m0 of ceil(amax / 448) from amax's bits: 448 = 1.75 * 2^8, so a mantissa
|
||||
# above 0.75 (0x600000) rounds the exponent up
|
||||
ea = amb >> fx.Int32(23)
|
||||
ex = (
|
||||
ea
|
||||
- fx.Int32(8)
|
||||
+ ((amb & fx.Int32(0x7FFFFF)) > fx.Int32(0x600000)).select(
|
||||
fx.Int32(1), fx.Int32(0)
|
||||
)
|
||||
)
|
||||
ex = (ex < fx.Int32(EX_FLOOR)).select(fx.Int32(EX_FLOOR), ex)
|
||||
return (ea >= fx.Int32(255)).select(fx.Int32(254), ex)
|
||||
|
||||
|
||||
def fp8_scaled(y, nexp):
|
||||
v = ldexp(y, nexp).maximumf(fx.Float32(-FP8_MAX))
|
||||
return (v > fx.Float32(FP8_MAX)).select(fx.Float32(FP8_MAX), v)
|
||||
|
||||
|
||||
def xcc_id():
|
||||
return fx.Int32(
|
||||
_llvm_d.InlineAsmOp(
|
||||
T.i32,
|
||||
[],
|
||||
"s_getreg_b32 $0, hwreg(HW_REG_XCC_ID)",
|
||||
"=s",
|
||||
has_side_effects=True,
|
||||
).res
|
||||
)
|
||||
|
||||
|
||||
def pin_v64(x):
|
||||
return fx.Int64(
|
||||
_llvm_d.InlineAsmOp(
|
||||
T.i64, [fx.Int64(x).ir_value()], "; pin $0", "=v,0", has_side_effects=True
|
||||
).res
|
||||
)
|
||||
|
||||
|
||||
def gload(addr_i64, n):
|
||||
return fx.generic_load(cu._to_ptr_global(addr_i64, fx.Float32, 4 * n), count=n)
|
||||
|
||||
|
||||
def inv_l1():
|
||||
_llvm_d.InlineAsmOp(None, [], "buffer_inv sc0", "", has_side_effects=True)
|
||||
|
||||
|
||||
def rmw_add64_agent(addr_i64, val):
|
||||
return fx.atomic_add(
|
||||
cu._to_ptr_global(addr_i64, fx.Int64, 8), fx.Int64(val), syncscope="agent"
|
||||
)
|
||||
|
||||
|
||||
def ld64_agent(addr_i64, volatile=True):
|
||||
return fx.generic_load(
|
||||
cu._to_ptr_global(addr_i64, fx.Int64, 8),
|
||||
volatile=volatile,
|
||||
memory_order=fx.AtomicOrdering.Monotonic,
|
||||
syncscope=fx.rocdl.SyncScope.AgentOneAs,
|
||||
)
|
||||
|
||||
|
||||
def st_raw(addr_i64, value, align):
|
||||
fx.generic_store(cu._to_ptr_global(addr_i64, alignment=align), value)
|
||||
|
||||
|
||||
def st_agent(addr_i64, value, align):
|
||||
fx.generic_store(
|
||||
cu._to_ptr_global(addr_i64, alignment=align),
|
||||
value,
|
||||
memory_order=fx.AtomicOrdering.Monotonic,
|
||||
syncscope=fx.rocdl.SyncScope.AgentOneAs,
|
||||
)
|
||||
|
||||
|
||||
def st_agent_vec(addr_i64, value, nbytes):
|
||||
# sc1 store: written through to the device coherence point (no L2 write-back
|
||||
# needed); s_nop 1 covers the store-data VGPR hazard the backend does not see
|
||||
if nbytes == 4:
|
||||
v = fx.Float32(value).bitcast(fx.Int32)
|
||||
op = "global_store_dword $0, $1, off sc1\n\ts_nop 1"
|
||||
else:
|
||||
assert nbytes == 16, nbytes
|
||||
v = fx.Vector(value).bitcast(fx.Int32)
|
||||
op = "global_store_dwordx4 $0, $1, off sc1\n\ts_nop 1"
|
||||
_llvm_d.InlineAsmOp(
|
||||
None,
|
||||
[fx.Int64(addr_i64).ir_value(), _arith.unwrap(v)],
|
||||
op,
|
||||
"v,v,~{memory}",
|
||||
has_side_effects=True,
|
||||
)
|
||||
|
||||
|
||||
def ld_agent_f32(addr_i64, n):
|
||||
if n == 1:
|
||||
return ld_agent_i32(addr_i64, volatile=False).bitcast(fx.Float32)
|
||||
parts = [
|
||||
ld64_agent(addr_i64 + fx.Int64(8 * i), volatile=False) for i in range(n // 2)
|
||||
]
|
||||
return fx.Vector.from_elements(parts, fx.Int64).bitcast(fx.Float32)
|
||||
|
||||
|
||||
@flyc.jit
|
||||
def spin_all_arrived(addr_i64, full, bit):
|
||||
cur = ld64_agent(addr_i64)
|
||||
while ((cur & full) != full) & ((cur & bit) != fx.Int64(0)):
|
||||
cur = ld64_agent(addr_i64)
|
||||
return cur
|
||||
|
||||
|
||||
def ld_agent_i32(addr_i64, volatile=True):
|
||||
return fx.generic_load(
|
||||
cu._to_ptr_global(addr_i64),
|
||||
volatile=volatile,
|
||||
memory_order=fx.AtomicOrdering.Monotonic,
|
||||
syncscope=fx.rocdl.SyncScope.AgentOneAs,
|
||||
)
|
||||
|
||||
|
||||
def tickets_state(v, want):
|
||||
z = fx.Int64(fx.rocdl.ballot(T.i64, (v == fx.Int32(0)).ir_value()))
|
||||
o = fx.Int64(fx.rocdl.ballot(T.i64, (v != want).ir_value()))
|
||||
return (z == fx.Int64(0)).select(fx.Int32(1), fx.Int32(0)) | (
|
||||
(o == fx.Int64(0)).select(fx.Int32(2), fx.Int32(0))
|
||||
)
|
||||
|
||||
|
||||
@flyc.jit
|
||||
def spin_tickets_state(addr_lane_i64, v, max_polls, want):
|
||||
r = tickets_state(v, want)
|
||||
k = fx.Int32(0)
|
||||
while ((r & fx.Int32(1)) == fx.Int32(0)) & (k < fx.Int32(max_polls)):
|
||||
r = tickets_state(ld_agent_i32(addr_lane_i64), want)
|
||||
k = k + fx.Int32(1)
|
||||
return r
|
||||
+5271
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,718 @@
|
||||
# SPDX-License-Identifier: MIT
|
||||
# Copyright (C) 2024-2026, Advanced Micro Devices, Inc. All rights reserved.
|
||||
#
|
||||
# Vendored from ROCm/aiter#5882 (https://github.com/ROCm/aiter/pull/5882),
|
||||
# commit 03359c43df (03359c43df40abf91ac38a2040c9b8e284d92297),
|
||||
# file aiter/ops/flydsl/mla_v4_decode.py.
|
||||
# Unchanged except for import paths and sglang formatting (isort, ruff format).
|
||||
# TODO(aiter#5882): remove once ROCm/aiter#5882 is in the aiter pin
|
||||
|
||||
"""FlyDSL DeepSeek-V4 fp8 MLA decode (v4 nm contract) on gfx950.
|
||||
|
||||
flydsl_mla_decode_fwd_v4_nm: drop-in for aiter.mla.mla_decode_fwd_v4_nm (bf16 output).
|
||||
flydsl_mla_v4_decode_fused: the same attention + inverse RoPE + wo_a mxfp8 quant.
|
||||
|
||||
The split plan is keyed by the batch size, head count, max_seqlen_q and a stream hint,
|
||||
never by kv lengths (graph-capture safe). The merge scratch is owned here, one per
|
||||
(device, stream).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
from functools import cache
|
||||
|
||||
import flydsl.expr as fx
|
||||
import torch
|
||||
from aiter.jit.utils.chip_info import get_gfx
|
||||
from aiter.ops.flydsl.kernels.tensor_shim import _run_compiled, ptr_arg
|
||||
|
||||
from .kernels.mla_v4_decode_gfx950 import (
|
||||
DV,
|
||||
MAX_GROUPS,
|
||||
MAX_S,
|
||||
ROPE,
|
||||
ROWB,
|
||||
WS_HDR,
|
||||
V4Cfg,
|
||||
build_mla_v4_decode,
|
||||
)
|
||||
from .mla_v4_grouped import GROUPED_MAX_MSQ
|
||||
|
||||
__all__ = [
|
||||
"flydsl_mla_decode_fwd_v4_nm",
|
||||
"flydsl_mla_v4_decode_fused",
|
||||
"flydsl_mla_v4_decode_supported",
|
||||
"flydsl_mla_v4_grouped_pays",
|
||||
]
|
||||
|
||||
# (H, max_seqlen_q) pairs the asm v4 nm kernel ships
|
||||
SUPPORTED = ((16, 1), (16, 2), (16, 4), (32, 1), (64, 1), (128, 1))
|
||||
|
||||
# Split plans measured on MI355X (cold caches, vs the best asm split count + merge).
|
||||
# H16 msq1 with a stream hint: T_pad upper bound -> (CSA, HCA) splits (HCA past
|
||||
# T_pad 40: _plan_h16's key-balanced grid)
|
||||
_POLICY_H16 = {
|
||||
8: (18, 18),
|
||||
16: (12, 12),
|
||||
24: (9, 9),
|
||||
32: (8, 8),
|
||||
40: (6, 6),
|
||||
48: (4, 4),
|
||||
64: (4, 4),
|
||||
80: (3, 3),
|
||||
128: (2, 2),
|
||||
160: (3, 1),
|
||||
256: (1, 1),
|
||||
}
|
||||
# below _POLICY_Q16: B upper bound -> (16-row blocks per CTA, splits) of K SPREAD plans
|
||||
_POLICY_WIDE = {
|
||||
(16, 2): ((1, 1, 32), (None, 1, 16)),
|
||||
(16, 4): ((None, 1, 16),),
|
||||
(32, 1): ((1, 1, 24), (None, 1, 16)),
|
||||
(64, 1): ((None, 1, 16),),
|
||||
(128, 1): ((None, 1, 16),),
|
||||
}
|
||||
# stream-length bodies (V4Cfg.SUB): B upper bound -> longest short stream (K, S = 1);
|
||||
# (H, msq) -> ((B max, B min, ((RB, S, longest stream), ...)), ...) mid-length streams
|
||||
_SHORT_KEYS = ((8, 192), (16, 256), (64, 768), (128, 256))
|
||||
# Q16 mains: 32-row ones take the short body up to _Q16_SHORT_KEYS keys; 64-row ones
|
||||
# past B 16 at S >= 3 run streams of <= max(_Q16_RB2_KEYS / S, _Q16_RB2_MIN) keys on
|
||||
# unsplit 32-row Q16 CTAs (at S = 2 the joint kernel spills: long streams pay more)
|
||||
_Q16_SHORT_KEYS = 160
|
||||
_Q16_RB2_KEYS, _Q16_RB2_MIN = 2304, 448
|
||||
_POLICY_MID = {
|
||||
(16, 2): ((1, 1, ((1, 16, 2048),)), (16, 9, ((1, 8, 4096),))),
|
||||
(16, 4): ((8, 5, ((1, 8, 3072),)), (16, 9, ((1, 4, 2048),))),
|
||||
(32, 1): ((16, 9, ((1, 8, 4096),)),),
|
||||
(64, 1): ((8, 5, ((1, 8, 3072),)), (16, 9, ((1, 4, 2048),))),
|
||||
}
|
||||
# smallest B of each pair on Q16 CTAs
|
||||
_POLICY_Q16 = {(16, 2): 9, (16, 4): 5, (32, 1): 9, (64, 1): 5, (128, 1): 2}
|
||||
# grids a little past half the GPU run 2 splits: largest grid, in 1/32 of the CUs, for
|
||||
# Q16 (RB, NRG) and for the H16 K layout
|
||||
_PAST_HALF = {(4, 1): 18, (2, 1): 19}
|
||||
_PAST_HALF_K = 19
|
||||
# Q16 split count -> streams of at most this many 32-key tiles run unsplit (SHORT)
|
||||
_Q16_SHORT_TILES = {2: 24, 3: 10}
|
||||
# unsplit Q16 grids of fewer CTAs than CUs: split 1 of the longest streams of > _IDLE_F
|
||||
# tiles on the idle CUs (V4Cfg.IDLE), past them on _IDLE_X2 more CTAs per XCD for
|
||||
# streams of > _IDLE_W tiles; larger grids of two row groups (up to 512 sequences):
|
||||
# splits 1-3 of streams of > _IDLE_WL tiles on _IDLE_XL CTAs per XCD past the grid
|
||||
_IDLE_X2, _IDLE_F, _IDLE_W = 4, 32, 96
|
||||
_IDLE_XL, _IDLE_WL = 16, 64
|
||||
# PV16R key window: (lo, hi at B <= CUs, hi above)
|
||||
_PV16_LO, _PV16_HI = 256, (4096, 2048)
|
||||
# grouped HCA: streams of <= 17 32-key tiles on the K body, longer ones on RQ
|
||||
_GROUPED_RQ_TILES = 18
|
||||
# H16 bf16 without a hint: most splits per sequence; longest unsplit stream; shortest
|
||||
# key-balanced SK run in 64-key tiles (grid of one, two CTAs per CU)
|
||||
_H16_MAX_S = 18
|
||||
_H16_SHORT_KEYS = 128
|
||||
_H16_SKT = (8, 12)
|
||||
# SK grids of at most this many splits per sequence: V4Cfg.TAIL where it adds some
|
||||
_H16_TAIL_SKD = 8
|
||||
# 64-row Q16 with fewer splits than this per sequence: V4Cfg.TAIL where it adds some
|
||||
_TAIL_S = 16
|
||||
|
||||
|
||||
@cache
|
||||
def _num_cus(device_index: int) -> int:
|
||||
return torch.cuda.get_device_properties(device_index).multi_processor_count
|
||||
|
||||
|
||||
def flydsl_mla_v4_decode_supported(
|
||||
num_heads: int, max_seqlen_q: int = 1, grouped: bool = False
|
||||
) -> bool:
|
||||
"""True if the FlyDSL v4 decode covers (num_heads, max_seqlen_q) on this GPU;
|
||||
grouped: with q_kv_bounds (H = 16, max_seqlen_q <= 8)."""
|
||||
try:
|
||||
if get_gfx() != "gfx950":
|
||||
return False
|
||||
except Exception: # noqa: BLE001
|
||||
return False
|
||||
if grouped:
|
||||
return num_heads == 16 and 1 <= max_seqlen_q <= GROUPED_MAX_MSQ
|
||||
return (num_heads, max_seqlen_q) in SUPPORTED
|
||||
|
||||
|
||||
def flydsl_mla_v4_grouped_pays(
|
||||
num_heads: int,
|
||||
num_draft: int,
|
||||
num_requests: int,
|
||||
hint: str | None = "hca",
|
||||
epi: str = "invrope_mxfp8",
|
||||
) -> bool:
|
||||
"""Whether a DSpark verify step should share one stream per request (grouped
|
||||
q_kv_bounds) on its HCA / SWA-only layers (MI355X, DSv4-Pro verify step)."""
|
||||
if num_heads != 16 or not 1 <= num_draft <= GROUPED_MAX_MSQ:
|
||||
return False
|
||||
if epi == "bf16" and hint in (None, "hca"):
|
||||
return num_requests >= 32
|
||||
return num_requests >= 64
|
||||
|
||||
|
||||
def _plan(
|
||||
H: int,
|
||||
msq: int,
|
||||
B: int,
|
||||
epi: str = "bf16",
|
||||
hint: str | None = None,
|
||||
cus: int = 256,
|
||||
grouped: bool = False,
|
||||
) -> V4Cfg:
|
||||
"""Variant for B sequences. hint: "csa" | "hca" | "swa" (SWA-only layer) | None."""
|
||||
cfg = _plan_for(H, msq, max(1, B), epi, hint, cus, grouped)
|
||||
if cfg.ws_need(B)[0] > MAX_GROUPS:
|
||||
# more merge groups than protocol slots: run unsplit
|
||||
cfg = dataclasses.replace(
|
||||
cfg,
|
||||
S=1,
|
||||
ADAPT=0,
|
||||
MINT=1,
|
||||
SUB=(),
|
||||
SPREAD=False,
|
||||
SHORT=0,
|
||||
SHORTM=False,
|
||||
SHORTF=0,
|
||||
OWN=False,
|
||||
SK=0,
|
||||
SKMEAN=False,
|
||||
IDLE=(),
|
||||
)
|
||||
return cfg
|
||||
|
||||
|
||||
def _plan_for(H, msq, B, epi, hint, cus, grouped) -> V4Cfg:
|
||||
if grouped:
|
||||
return _plan_grouped(msq, B, epi, hint, cus)
|
||||
if H == 16 and msq == 1:
|
||||
return _plan_h16(B, epi, hint, cus)
|
||||
if B >= _POLICY_Q16[(H, msq)]:
|
||||
return _with_subs(_q16_plan(H, msq, B, epi, cus), B, cus)
|
||||
for bmax, rb, s in _POLICY_WIDE[(H, msq)]:
|
||||
if bmax is None or B <= bmax:
|
||||
break
|
||||
cfg = V4Cfg(H=H, MSQ=msq, RB=rb, S=s, EPI=epi, KC=True, SPREAD=True)
|
||||
return _with_subs(cfg, B, cus)
|
||||
|
||||
|
||||
def _plan_h16(B, epi, hint, cus) -> V4Cfg:
|
||||
tpad = -(-B // 8) * 8
|
||||
adapt = {}
|
||||
if hint == "swa":
|
||||
s = 1
|
||||
elif epi == "bf16" and hint is None:
|
||||
s = min(_H16_MAX_S, max(1, cus // tpad))
|
||||
if 16 <= tpad <= 2 * cus:
|
||||
# ragged batches: the key-balanced grid (V4Cfg.SK; past cus sequences two
|
||||
# CTAs per CU)
|
||||
sk = cus * -(-tpad // cus)
|
||||
cfg = _sk_plan(B, epi, cus, 16 * cus // sk, SK=sk, SKT=_H16_SKT[sk > cus])
|
||||
if 2 * tpad > cus and tpad <= cus * _PAST_HALF_K // 32:
|
||||
cfg = dataclasses.replace(cfg, SKD=2, SKT=14, SKMEAN=True, SHORT=4)
|
||||
return cfg
|
||||
if tpad > cus:
|
||||
s = 2
|
||||
adapt = {"ADAPT": cus, "MINT": 16}
|
||||
elif tpad < 16:
|
||||
short = V4Cfg(H=16, RB=1, EPI=epi, KC=True)
|
||||
adapt = {"SUB": ((_H16_SHORT_KEYS, short),)}
|
||||
else:
|
||||
s = 1
|
||||
for tb, v in _POLICY_H16.items():
|
||||
if tpad <= tb:
|
||||
s = v[0 if hint == "csa" else 1]
|
||||
break
|
||||
if hint == "hca" and tpad > 40:
|
||||
# ragged HCA batches (a few long sessions set the longest stream): the
|
||||
# key-balanced grid; past one sequence per CU, run-time split counts
|
||||
if tpad > cus:
|
||||
return V4Cfg(H=16, RB=1, S=8, EPI=epi, KC=True, ADAPT=cus, MINT=4)
|
||||
return _sk_plan(B, epi, cus, 8, SK=cus, SKT=6)
|
||||
return V4Cfg(H=16, RB=1, S=min(s, MAX_S), EPI=epi, KC=True, **adapt)
|
||||
|
||||
|
||||
def _sk_plan(B, epi, cus, S, **kw) -> V4Cfg:
|
||||
"""H16 key-balanced grid (V4Cfg.SK) of at most max(S, SKD) splits, else SKD
|
||||
splits per sequence: cus // T_pad, or cus // B on V4Cfg.TAIL where that is more"""
|
||||
tpad = -(-B // 8) * 8
|
||||
skd = max(1, min(_H16_MAX_S, cus // tpad))
|
||||
tail = skd < cus // B <= _H16_TAIL_SKD
|
||||
skd = cus // B if tail else skd
|
||||
return V4Cfg(
|
||||
H=16,
|
||||
RB=1,
|
||||
EPI=epi,
|
||||
KC=True,
|
||||
S=max(S, skd),
|
||||
SKD=skd,
|
||||
TAIL=tail,
|
||||
MINT=3 if 4 * tpad > cus else 1,
|
||||
SHORT=_H16_SHORT_KEYS // 64,
|
||||
**kw,
|
||||
)
|
||||
|
||||
|
||||
def _with_subs(cfg: V4Cfg, B: int, cus: int) -> V4Cfg:
|
||||
"""cfg + its stream-length bodies where they fit (at most the sequence's CTAs; the
|
||||
short body at most one CTA per CU over the batch; under Q16 16-row K or 32-row Q16
|
||||
bodies)."""
|
||||
if cfg.S == 1 or cfg.XR or cfg.IDLE:
|
||||
return cfg
|
||||
if cfg.Q16 and cfg.RB == 4 and B > 16 and cfg.S >= 3:
|
||||
rb2 = dataclasses.replace(cfg, RB=2, S=1, SHORT=0, PF16=False)
|
||||
keys = max(_Q16_RB2_KEYS // cfg.S, _Q16_RB2_MIN)
|
||||
# (the main's short streams all take the body)
|
||||
return dataclasses.replace(cfg, SHORT=0, SUB=((keys, rb2),))
|
||||
subs = []
|
||||
keys = next((k for bmax, k in _SHORT_KEYS if B <= bmax), 0)
|
||||
if cfg.Q16 and cfg.RB == 2:
|
||||
keys = min(keys, _Q16_SHORT_KEYS)
|
||||
rbs = (1,) if cfg.Q16 else (1, 2)
|
||||
for rb in rbs if keys else ():
|
||||
nrg = cfg.H * cfg.MSQ // (16 * rb)
|
||||
if nrg <= cfg.NRG * cfg.S and B * nrg <= cus:
|
||||
subs.append((keys, _k_body(cfg, rb, 1)))
|
||||
break
|
||||
for bmax, bmin, mids in _POLICY_MID.get((cfg.H, cfg.MSQ), ()):
|
||||
if bmin <= B <= bmax:
|
||||
for rb, s, mkeys in mids:
|
||||
sub = _k_body(cfg, rb, s)
|
||||
fits = rb in rbs and sub.NRG * s <= cfg.NRG * cfg.S and sub != cfg
|
||||
if fits and mkeys > keys:
|
||||
subs.append((mkeys, sub))
|
||||
keys = mkeys
|
||||
break
|
||||
return dataclasses.replace(cfg, SUB=tuple(subs)) if subs else cfg
|
||||
|
||||
|
||||
def _q16_plan(H: int, msq: int, B: int, epi: str, cus: int) -> V4Cfg:
|
||||
rb = 4 if H * msq % 64 == 0 else 2
|
||||
nrg = H * msq // (16 * rb)
|
||||
bpad = -(-B // 8) * 8
|
||||
s = max(1, min(MAX_S, cus // (bpad * nrg)))
|
||||
xr = nrg > 1 and bpad <= 8
|
||||
if nrg > 1 and bpad > 8:
|
||||
# row groups on consecutive XCDs where that allows more splits
|
||||
s_xr = min(MAX_S, cus // 8 // -(-B * nrg // 8))
|
||||
if s_xr > s:
|
||||
s, xr = s_xr, True
|
||||
# the units past the last complete XCD octet over all XCDs (V4Cfg.TAIL)
|
||||
tail_ok = rb == 4 and s < _TAIL_S and 8 < B < cus // 8 * nrg and B % 8
|
||||
s_tl = _tail_splits(B, nrg, xr, cus) if tail_ok else 0
|
||||
tail, s = s_tl > s, max(s, s_tl)
|
||||
mean_short = rb == 4 and nrg == 1 and 2 * bpad == cus
|
||||
past_half = (
|
||||
s == 1
|
||||
and not xr
|
||||
and cus < 2 * bpad * nrg
|
||||
and bpad * nrg <= cus * _PAST_HALF.get((rb, nrg), 0) // 32
|
||||
)
|
||||
if past_half:
|
||||
s = 2
|
||||
if s == 1 and not xr and (bpad * nrg < cus or nrg > 1 and bpad <= 512):
|
||||
s, idle = 2, (cus // 8, _IDLE_X2, _IDLE_F, _IDLE_W)
|
||||
if bpad * nrg >= cus:
|
||||
s, idle = 4, (bpad // 8 * nrg, _IDLE_XL, _IDLE_F, _IDLE_WL)
|
||||
return V4Cfg(
|
||||
H=H,
|
||||
MSQ=msq,
|
||||
RB=rb,
|
||||
S=s,
|
||||
EPI=epi,
|
||||
LAYOUT="RD",
|
||||
Q16=True,
|
||||
PF16=True,
|
||||
PFN=cus if bpad * nrg > cus else 0,
|
||||
IDLE=idle,
|
||||
)
|
||||
if s == 1:
|
||||
short = 0
|
||||
elif mean_short:
|
||||
short = 64
|
||||
elif past_half:
|
||||
short = 1 << 16
|
||||
elif bpad <= 8:
|
||||
short = 6
|
||||
else:
|
||||
short = _Q16_SHORT_TILES.get(s, 8)
|
||||
cfg = V4Cfg(
|
||||
H=H,
|
||||
MSQ=msq,
|
||||
RB=rb,
|
||||
S=s,
|
||||
EPI=epi,
|
||||
LAYOUT="RD",
|
||||
Q16=True,
|
||||
SHORT=short,
|
||||
SHORTM=(mean_short or past_half) and s > 1,
|
||||
SHORTF=16 if past_half else 0,
|
||||
PF16=s > 1,
|
||||
XR=xr,
|
||||
PFN=cus if bpad * nrg > cus else 0,
|
||||
TAIL=tail,
|
||||
)
|
||||
if rb == 4 and nrg == 1 and bpad >= cus and epi == "bf16":
|
||||
cfg = dataclasses.replace(
|
||||
cfg, PV16R=(_PV16_LO, _PV16_HI[0] if bpad <= cus else _PV16_HI[1])
|
||||
)
|
||||
return cfg
|
||||
|
||||
|
||||
def _tail_splits(B: int, nrg: int, xr: bool, cus: int) -> int:
|
||||
# most splits under V4Cfg.TAIL with at most one CTA per CU on every XCD (one row
|
||||
# group: 7/8 of them; a full XCD costs up to 1.25x at 1k-4k keys; XR: at most 12,
|
||||
# more cost ~8 % at 200-500 keys)
|
||||
units, per = (B * nrg, 1) if xr else (B, nrg)
|
||||
cap = cus // 8 if nrg > 1 else cus * 7 // 64
|
||||
s = 12 if xr else MAX_S
|
||||
while s > 1 and units // 8 * s * per + -(-(units % 8) * s * per // 8) > cap:
|
||||
s -= 1
|
||||
return s
|
||||
|
||||
|
||||
def _k_body(cfg: V4Cfg, rb: int, s: int) -> V4Cfg:
|
||||
return V4Cfg(
|
||||
H=cfg.H,
|
||||
MSQ=cfg.MSQ,
|
||||
RB=rb,
|
||||
S=s,
|
||||
EPI=cfg.EPI,
|
||||
KC=True,
|
||||
SPREAD=cfg.SPREAD,
|
||||
TAIL=cfg.TAIL,
|
||||
)
|
||||
|
||||
|
||||
def _plan_grouped(msq: int, B: int, epi: str, hint: str | None, cus: int) -> V4Cfg:
|
||||
"""Grouped verify at H = 16: B groups of up to msq drafts sharing one stream."""
|
||||
hca = hint in (None, "hca")
|
||||
if epi == "bf16" and (B > 32 or (B == 32 and hca)):
|
||||
return _grouped_q16(msq, B, hca, cus)
|
||||
if B >= 32 and hca:
|
||||
# long streams on one 128-row RQ CTA per (group, split), short ones on K
|
||||
short = V4Cfg(
|
||||
H=16,
|
||||
MSQ=GROUPED_MAX_MSQ,
|
||||
RB=2 if B >= 64 else 1,
|
||||
EPI=epi,
|
||||
KC=True,
|
||||
MASK="bounds",
|
||||
)
|
||||
return V4Cfg(
|
||||
H=16,
|
||||
MSQ=GROUPED_MAX_MSQ,
|
||||
RB=GROUPED_MAX_MSQ,
|
||||
S=4,
|
||||
EPI=epi,
|
||||
LAYOUT="RQ",
|
||||
MASK="bounds",
|
||||
OWN=True,
|
||||
SUB=((32 * (_GROUPED_RQ_TILES - 1), short),),
|
||||
)
|
||||
if B > 32:
|
||||
return V4Cfg(
|
||||
H=16,
|
||||
MSQ=msq + (msq & 1),
|
||||
RB=2,
|
||||
EPI=epi,
|
||||
KC=hint != "swa",
|
||||
MASK="bounds",
|
||||
)
|
||||
s = 1 if hint == "swa" or B > 16 else (2 if B > 8 else 4)
|
||||
return V4Cfg(H=16, MSQ=msq, RB=1, S=s, EPI=epi, MASK="bounds")
|
||||
|
||||
|
||||
def _grouped_q16(msq: int, B: int, hca: bool, cus: int) -> V4Cfg:
|
||||
"""32-row Q16 CTAs (two drafts each), 64-row ones where the 32-row grid exceeds
|
||||
one CTA per CU and they need fewer."""
|
||||
mp = msq + (msq & 1)
|
||||
rb = 2
|
||||
nrg = 16 * mp // 32
|
||||
bpad = -(-B // 8) * 8
|
||||
mp4 = -(-msq // 4) * 4
|
||||
if bpad * nrg > cus and 16 * mp4 // 64 < nrg:
|
||||
mp, rb, nrg = mp4, 4, 16 * mp4 // 64
|
||||
s = max(1, min(4, cus // (bpad * nrg))) if hca else 1
|
||||
return V4Cfg(
|
||||
H=16,
|
||||
MSQ=mp,
|
||||
RB=rb,
|
||||
S=s,
|
||||
LAYOUT="RD",
|
||||
MASK="bounds",
|
||||
Q16=True,
|
||||
SHORT=16 if s > 1 else 0,
|
||||
PF16=s > 1,
|
||||
)
|
||||
|
||||
|
||||
class _Workspace:
|
||||
"""Grow-only merge scratch of one (device, stream): protocol slots (hdr, zero
|
||||
between launches) + split partials (buf). A new stream takes a spare hdr zeroed
|
||||
at an earlier eager launch, so a graph captured on it replays no memset. Old
|
||||
buffers stay alive: a captured graph may still reference them."""
|
||||
|
||||
def __init__(self, device):
|
||||
self.device = device
|
||||
self.hdr = None
|
||||
self.buf = None
|
||||
self._keep = []
|
||||
|
||||
def ensure(self, nbytes: int):
|
||||
if self.hdr is None:
|
||||
self.hdr = _HDR_SPARE.pop(self.device.index, None)
|
||||
if self.hdr is None:
|
||||
self.hdr = torch.zeros(
|
||||
WS_HDR // 4, dtype=torch.int32, device=self.device
|
||||
)
|
||||
if (
|
||||
not torch.cuda.is_current_stream_capturing()
|
||||
and self.device.index not in _HDR_SPARE
|
||||
):
|
||||
_HDR_SPARE[self.device.index] = _zeroed_hdr(self.device)
|
||||
need = -(-nbytes // 4)
|
||||
if self.buf is None or self.buf.numel() < need:
|
||||
if self.buf is not None:
|
||||
self._keep.append(self.buf)
|
||||
need = max(need, self.buf.numel() * 2)
|
||||
self.buf = torch.empty(need, dtype=torch.int32, device=self.device)
|
||||
return self
|
||||
|
||||
|
||||
_WORKSPACES: dict = {}
|
||||
_HDR_SPARE: dict = {}
|
||||
_ZERO_STREAMS: dict = {}
|
||||
_COMPILED: set = set()
|
||||
_DUMMY: dict = {}
|
||||
|
||||
|
||||
def _zeroed_hdr(device):
|
||||
# zeroed on an idle side stream, before any stream can use it
|
||||
zs = _ZERO_STREAMS.get(device.index)
|
||||
if zs is None:
|
||||
zs = _ZERO_STREAMS[device.index] = torch.cuda.Stream(device)
|
||||
with torch.cuda.stream(zs):
|
||||
hdr = torch.zeros(WS_HDR // 4, dtype=torch.int32, device=device)
|
||||
zs.synchronize()
|
||||
return hdr
|
||||
|
||||
|
||||
def _workspace(device) -> _Workspace:
|
||||
stream = torch.cuda.current_stream(device)
|
||||
key = (device.index, stream.cuda_stream)
|
||||
ws = _WORKSPACES.get(key)
|
||||
if ws is None:
|
||||
ws = _WORKSPACES[key] = _Workspace(device)
|
||||
return ws
|
||||
|
||||
|
||||
def _launch(
|
||||
cfg: V4Cfg,
|
||||
B,
|
||||
q,
|
||||
q_rope,
|
||||
kv,
|
||||
kv_rope,
|
||||
qo_indptr,
|
||||
kv_indptr,
|
||||
kv_indices,
|
||||
sink,
|
||||
out,
|
||||
out_s0,
|
||||
out_s1,
|
||||
xs=None,
|
||||
positions=None,
|
||||
freqs=None,
|
||||
bounds=None,
|
||||
):
|
||||
device = q.device
|
||||
dmy = _DUMMY.get(device.index)
|
||||
if dmy is None:
|
||||
dmy = _DUMMY[device.index] = torch.zeros(64, dtype=torch.int64, device=device)
|
||||
groups, nbytes = cfg.ws_need(B)
|
||||
if groups:
|
||||
ws = _workspace(device).ensure(nbytes)
|
||||
hdr, wsb = ws.hdr, ws.buf
|
||||
else:
|
||||
hdr = wsb = dmy
|
||||
if cfg not in _COMPILED and torch.cuda.is_current_stream_capturing():
|
||||
raise RuntimeError(
|
||||
f"flydsl_mla_v4_decode: variant {cfg} not compiled; run this batch size "
|
||||
"once eagerly before graph capture"
|
||||
)
|
||||
u8 = torch.uint8
|
||||
epi = cfg.EPI == "invrope_mxfp8"
|
||||
_run_compiled(
|
||||
build_mla_v4_decode(cfg),
|
||||
ptr_arg(q.view(u8), fx.Uint8),
|
||||
ptr_arg(q_rope, fx.BFloat16),
|
||||
ptr_arg(kv.view(u8), fx.Uint8),
|
||||
ptr_arg(kv_rope, fx.BFloat16),
|
||||
ptr_arg(dmy if qo_indptr is None else qo_indptr, fx.Int32),
|
||||
ptr_arg(kv_indptr, fx.Int32),
|
||||
ptr_arg(kv_indices, fx.Int32),
|
||||
ptr_arg(sink, fx.Float32),
|
||||
ptr_arg(out.view(u8) if epi else out, fx.Uint8 if epi else fx.BFloat16),
|
||||
ptr_arg(dmy if xs is None else xs.view(u8), fx.Uint8),
|
||||
ptr_arg(dmy if positions is None else positions, fx.Int64),
|
||||
ptr_arg(dmy if freqs is None else freqs, fx.Float32),
|
||||
ptr_arg(wsb, fx.Int32),
|
||||
ptr_arg(dmy if bounds is None else bounds, fx.Int32),
|
||||
ptr_arg(hdr, fx.Int32),
|
||||
B,
|
||||
kv.shape[0],
|
||||
out_s0,
|
||||
out_s1,
|
||||
fx.Stream(torch.cuda.current_stream(device)),
|
||||
)
|
||||
_COMPILED.add(cfg)
|
||||
|
||||
|
||||
def _check_bounds(q_kv_bounds, total_q):
|
||||
if q_kv_bounds is not None:
|
||||
assert q_kv_bounds.dtype == torch.int32 and q_kv_bounds.is_contiguous()
|
||||
assert q_kv_bounds.dim() == 2 and q_kv_bounds.shape[1] == 4
|
||||
assert q_kv_bounds.shape[0] >= total_q, (q_kv_bounds.shape, total_q)
|
||||
|
||||
|
||||
def flydsl_mla_decode_fwd_v4_nm(
|
||||
q,
|
||||
qrope,
|
||||
kv_buffer,
|
||||
kvrope,
|
||||
output,
|
||||
qo_indptr,
|
||||
kv_indptr,
|
||||
kv_page_indices,
|
||||
max_seqlen_q,
|
||||
*,
|
||||
sink,
|
||||
hint=None,
|
||||
q_kv_bounds=None,
|
||||
):
|
||||
"""aiter.mla.mla_decode_fwd_v4_nm on gfx950: writes the final bf16 attention into
|
||||
``output`` [total_q, H, 512] (rows without keys become NaN).
|
||||
|
||||
hint: "csa" | "hca" | "swa" | None, only selects the split plan.
|
||||
q_kv_bounds: optional int32 [total_q, 4] (a0, a1, b0, b1); q row r attends stream
|
||||
position j of its sequence iff a0 <= j < a1 or b0 <= j < b1 (grouped verify).
|
||||
"""
|
||||
H = q.shape[1]
|
||||
msq = int(max_seqlen_q)
|
||||
grouped = q_kv_bounds is not None
|
||||
if not flydsl_mla_v4_decode_supported(H, msq, grouped):
|
||||
raise RuntimeError(
|
||||
f"flydsl_mla_decode_fwd_v4_nm: no variant for num_heads={H}, "
|
||||
f"max_seqlen_q={msq}, grouped={grouped}"
|
||||
)
|
||||
B = qo_indptr.numel() - 1
|
||||
assert q.dim() == 3 and q.shape[2] == ROWB and q.element_size() == 1, q.shape
|
||||
assert qrope.shape[1:] == (H, ROPE) and qrope.dtype == torch.bfloat16
|
||||
assert q.is_contiguous() and qrope.is_contiguous()
|
||||
assert kv_buffer.element_size() == 1 and kvrope.dtype == torch.bfloat16
|
||||
kv = kv_buffer.reshape(-1, ROWB)
|
||||
kv_rope = kvrope.reshape(-1, ROPE)
|
||||
assert kv.shape[0] == kv_rope.shape[0]
|
||||
assert sink.dtype == torch.float32 and sink.is_contiguous() and sink.numel() == H
|
||||
for t in (qo_indptr, kv_indptr, kv_page_indices):
|
||||
assert t.dtype == torch.int32 and t.is_contiguous(), t.dtype
|
||||
assert output.dtype == torch.bfloat16 and output.shape[1:] == (H, DV)
|
||||
assert output.stride(2) == 1
|
||||
_check_bounds(q_kv_bounds, q.shape[0])
|
||||
cfg = _plan(H, msq, B, "bf16", hint, _num_cus(q.device.index), grouped)
|
||||
if B > 0:
|
||||
_launch(
|
||||
cfg,
|
||||
B,
|
||||
q,
|
||||
qrope,
|
||||
kv,
|
||||
kv_rope,
|
||||
qo_indptr,
|
||||
kv_indptr,
|
||||
kv_page_indices,
|
||||
sink,
|
||||
output,
|
||||
output.stride(0),
|
||||
output.stride(1),
|
||||
bounds=q_kv_bounds,
|
||||
)
|
||||
return output
|
||||
|
||||
|
||||
def flydsl_mla_v4_decode_fused(
|
||||
q,
|
||||
q_rope,
|
||||
kv,
|
||||
kv_rope,
|
||||
kv_indptr,
|
||||
kv_indices,
|
||||
sink,
|
||||
positions,
|
||||
freqs,
|
||||
compress_ratio,
|
||||
*,
|
||||
qo_indptr=None,
|
||||
max_seqlen_q: int = 1,
|
||||
q_kv_bounds=None,
|
||||
):
|
||||
"""DeepSeek-V4 decode tail in one launch: attention + split merge + inverse RoPE of
|
||||
dims 448..511 + wo_a mxfp8 quant.
|
||||
|
||||
q / q_rope / kv / kv_rope / sink: as flydsl_mla_decode_fwd_v4_nm; one sequence per
|
||||
token (kv_indptr [T + 1]) unless q_kv_bounds + qo_indptr + max_seqlen_q give
|
||||
grouped-verify streams. positions [T] int64; freqs [max_pos, 64] fp32 =
|
||||
view_as_real(freqs_cis). compress_ratio: 4 CSA, 128 HCA, else SWA-only.
|
||||
Returns (xq [T, H*512] fp8_e4m3fn, xs [T, H*4] e8m0); tokens without keys get NaN.
|
||||
"""
|
||||
T, H = q.shape[0], q.shape[1]
|
||||
msq = int(max_seqlen_q)
|
||||
grouped = q_kv_bounds is not None
|
||||
if not (msq == 1 or grouped) or not flydsl_mla_v4_decode_supported(H, msq, grouped):
|
||||
raise RuntimeError(
|
||||
f"flydsl_mla_v4_decode_fused: no variant for num_heads={H}, "
|
||||
f"max_seqlen_q={msq}, grouped={grouped}"
|
||||
)
|
||||
if grouped:
|
||||
assert qo_indptr is not None and qo_indptr.dtype == torch.int32
|
||||
B = qo_indptr.numel() - 1
|
||||
else:
|
||||
B = T
|
||||
assert q.shape[2] == ROWB and q_rope.shape[1:] == (H, ROPE)
|
||||
assert q.is_contiguous() and q_rope.is_contiguous() and q.element_size() == 1
|
||||
assert q_rope.dtype == kv_rope.dtype == torch.bfloat16
|
||||
assert kv_indptr.dtype == kv_indices.dtype == torch.int32
|
||||
assert sink.dtype == freqs.dtype == torch.float32 and positions.dtype == torch.int64
|
||||
assert sink.is_contiguous() and freqs.is_contiguous() and kv.element_size() == 1
|
||||
assert freqs.shape[-1] == ROPE
|
||||
kv = kv.reshape(-1, ROWB)
|
||||
kv_rope = kv_rope.reshape(-1, ROPE)
|
||||
assert kv.shape[0] == kv_rope.shape[0]
|
||||
xq = torch.empty((T, H * DV), dtype=torch.float8_e4m3fn, device=q.device)
|
||||
xs = torch.empty((T, H * DV // 128), dtype=torch.uint8, device=q.device)
|
||||
_check_bounds(q_kv_bounds, T)
|
||||
hint = {4: "csa", 128: "hca"}.get(compress_ratio, "swa")
|
||||
cfg = _plan(H, msq, B, "invrope_mxfp8", hint, _num_cus(q.device.index), grouped)
|
||||
if T > 0 and B > 0:
|
||||
_launch(
|
||||
cfg,
|
||||
B,
|
||||
q,
|
||||
q_rope,
|
||||
kv,
|
||||
kv_rope,
|
||||
None if cfg.MSQ == 1 else qo_indptr,
|
||||
kv_indptr,
|
||||
kv_indices,
|
||||
sink,
|
||||
xq,
|
||||
0,
|
||||
0,
|
||||
xs,
|
||||
positions,
|
||||
freqs,
|
||||
q_kv_bounds,
|
||||
)
|
||||
return xq, xs
|
||||
@@ -0,0 +1,174 @@
|
||||
# SPDX-License-Identifier: MIT
|
||||
# Copyright (C) 2024-2026, Advanced Micro Devices, Inc. All rights reserved.
|
||||
#
|
||||
# Vendored from ROCm/aiter#5882 (https://github.com/ROCm/aiter/pull/5882),
|
||||
# commit 03359c43df (03359c43df40abf91ac38a2040c9b8e284d92297),
|
||||
# file aiter/ops/flydsl/mla_v4_grouped.py.
|
||||
# Unchanged except for import paths and sglang formatting (isort, ruff format).
|
||||
# TODO(aiter#5882): remove once ROCm/aiter#5882 is in the aiter pin
|
||||
|
||||
"""Grouped DSpark verify streams for the FlyDSL v4 decode.
|
||||
|
||||
The num_draft tokens of a request (positions p-nd+1 .. p) share one stream at an HCA /
|
||||
SWA-only layer: [compressed tail of the last draft (C entries)][SWA ring slots of
|
||||
positions p-n+1 .. p], n = min(p+1, win+nd-1) (the sglang#41120 layout). q_kv_bounds
|
||||
gives each draft its exact key set in it: [0, C_q) and its own window.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
__all__ = ["flydsl_mla_v4_grouped_verify_meta"]
|
||||
|
||||
GROUPED_MAX_MSQ = 8 # drafts per request
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _grouped_meta_kernel(
|
||||
idx_ptr,
|
||||
kv_indptr_ptr,
|
||||
qo_indptr_ptr,
|
||||
bnd_ptr,
|
||||
slot_ptr,
|
||||
pos_ptr,
|
||||
tail_len_ptr,
|
||||
tail_page_ptr,
|
||||
G,
|
||||
ring_stride,
|
||||
swa_pages,
|
||||
tail_stride,
|
||||
ND: tl.constexpr,
|
||||
WIN: tl.constexpr,
|
||||
TAIL_B: tl.constexpr,
|
||||
WIN_B: tl.constexpr,
|
||||
GB: tl.constexpr,
|
||||
HAS_TAIL: tl.constexpr,
|
||||
MAXND: tl.constexpr,
|
||||
):
|
||||
g = tl.program_id(0)
|
||||
# start of this group's stream: the previous groups' lengths
|
||||
base = 0
|
||||
for off in tl.range(0, G, GB):
|
||||
gg = off + tl.arange(0, GB)
|
||||
m = gg < g
|
||||
last = gg * ND + ND - 1
|
||||
p = tl.load(pos_ptr + last, mask=m, other=0).to(tl.int32)
|
||||
if HAS_TAIL:
|
||||
c = tl.load(tail_len_ptr + last, mask=m, other=0).to(tl.int32)
|
||||
else:
|
||||
c = tl.zeros([GB], tl.int32)
|
||||
n = tl.minimum(p + 1, WIN + ND - 1)
|
||||
base += tl.sum(tl.where(m, c + n, 0))
|
||||
first = g * ND
|
||||
last = first + ND - 1
|
||||
p = tl.load(pos_ptr + last).to(tl.int32)
|
||||
if HAS_TAIL:
|
||||
c = tl.load(tail_len_ptr + last).to(tl.int32)
|
||||
else:
|
||||
c = 0
|
||||
n = tl.minimum(p + 1, WIN + ND - 1)
|
||||
tl.store(kv_indptr_ptr + g + 1, base + c + n)
|
||||
tl.store(qo_indptr_ptr + g + 1, first + ND)
|
||||
if g == 0:
|
||||
tl.store(kv_indptr_ptr, 0)
|
||||
tl.store(qo_indptr_ptr, 0)
|
||||
if HAS_TAIL:
|
||||
for off in tl.range(0, c, TAIL_B):
|
||||
jt = off + tl.arange(0, TAIL_B)
|
||||
mt = jt < c
|
||||
pi = tl.load(
|
||||
tail_page_ptr + last.to(tl.int64) * tail_stride + jt, mask=mt, other=-1
|
||||
)
|
||||
tl.store(
|
||||
idx_ptr + base + jt, tl.where(pi >= 0, pi + swa_pages, -1), mask=mt
|
||||
)
|
||||
slot = tl.load(slot_ptr + last).to(tl.int64)
|
||||
i = tl.arange(0, WIN_B)
|
||||
ap = (p - n + 1 + i).to(tl.int64)
|
||||
tl.store(
|
||||
idx_ptr + base + c + i,
|
||||
(slot * ring_stride + ap % ring_stride).to(tl.int32),
|
||||
mask=i < n,
|
||||
)
|
||||
# per-row bounds, stream relative
|
||||
q = tl.arange(0, MAXND)
|
||||
mq = q < ND
|
||||
t = first + q
|
||||
pq = tl.load(pos_ptr + t, mask=mq, other=0).to(tl.int32)
|
||||
if HAS_TAIL:
|
||||
cq = tl.load(tail_len_ptr + t, mask=mq, other=0).to(tl.int32)
|
||||
else:
|
||||
cq = tl.zeros([MAXND], tl.int32)
|
||||
s0 = p - n + 1
|
||||
tb = bnd_ptr + t.to(tl.int64) * 4
|
||||
tl.store(tb + 0, tl.zeros([MAXND], tl.int32), mask=mq)
|
||||
tl.store(tb + 1, cq, mask=mq)
|
||||
tl.store(tb + 2, c + tl.maximum(0, pq - WIN + 1 - s0), mask=mq)
|
||||
tl.store(tb + 3, c + pq - s0 + 1, mask=mq)
|
||||
|
||||
|
||||
def flydsl_mla_v4_grouped_verify_meta(
|
||||
state_slot: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
tail_len: torch.Tensor | None,
|
||||
tail_page_indices: torch.Tensor | None,
|
||||
*,
|
||||
win: int,
|
||||
ring_stride: int,
|
||||
swa_pages: int,
|
||||
num_draft: int,
|
||||
):
|
||||
"""(kv_indices, kv_indptr, qo_indptr, q_kv_bounds) of the grouped streams, one
|
||||
graph-capturable launch; N = requests * num_draft tokens, request-major.
|
||||
|
||||
state_slot / positions [N]; tail_len [N] / tail_page_indices [N, Wc] int32: the
|
||||
committed compressed entries per token (None: SWA-only). kv_indices is sized for
|
||||
the worst case. Use with max_seqlen_q = num_draft.
|
||||
"""
|
||||
assert 1 <= num_draft <= GROUPED_MAX_MSQ
|
||||
dev = positions.device
|
||||
N = positions.shape[0]
|
||||
assert N % num_draft == 0, (N, num_draft)
|
||||
G = N // num_draft
|
||||
has_tail = tail_len is not None and tail_page_indices is not None
|
||||
Wc = tail_page_indices.shape[1] if has_tail else 0
|
||||
idx = torch.empty(
|
||||
max(G * (Wc + win + num_draft - 1), 1), dtype=torch.int32, device=dev
|
||||
)
|
||||
kv_indptr = torch.empty(G + 1, dtype=torch.int32, device=dev)
|
||||
qo_indptr = torch.empty(G + 1, dtype=torch.int32, device=dev)
|
||||
bnd = torch.empty((N, 4), dtype=torch.int32, device=dev)
|
||||
if G == 0:
|
||||
kv_indptr.zero_()
|
||||
qo_indptr.zero_()
|
||||
return idx, kv_indptr, qo_indptr, bnd
|
||||
assert state_slot.is_contiguous() and positions.is_contiguous()
|
||||
if has_tail:
|
||||
assert tail_len.is_contiguous() and tail_page_indices.stride(1) == 1
|
||||
tp = tail_page_indices if has_tail else positions
|
||||
_grouped_meta_kernel[(G,)](
|
||||
idx,
|
||||
kv_indptr,
|
||||
qo_indptr,
|
||||
bnd,
|
||||
state_slot,
|
||||
positions,
|
||||
tail_len if has_tail else positions,
|
||||
tp,
|
||||
G,
|
||||
ring_stride,
|
||||
swa_pages,
|
||||
tp.stride(0) if has_tail else 0,
|
||||
ND=num_draft,
|
||||
WIN=win,
|
||||
TAIL_B=min(1024, triton.next_power_of_2(max(Wc, 1))),
|
||||
WIN_B=triton.next_power_of_2(win + num_draft - 1),
|
||||
GB=min(1024, triton.next_power_of_2(G)),
|
||||
HAS_TAIL=has_tail,
|
||||
MAXND=GROUPED_MAX_MSQ,
|
||||
num_warps=4,
|
||||
)
|
||||
return idx, kv_indptr, qo_indptr, bnd
|
||||
@@ -27,6 +27,7 @@ on via ``topk_length``); the per-token compressed count is recovered from the
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from functools import lru_cache
|
||||
from typing import Optional, Tuple
|
||||
|
||||
@@ -49,6 +50,8 @@ from sglang.kernels.ops.attention.dsv4.unified_kv_kernels.paged_prefill import (
|
||||
sparse_attn_v4_paged_prefill,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# SWA ring scatter
|
||||
@@ -282,6 +285,289 @@ def _decode_fp8_tail_splits(T: int, compress_ratio: Optional[int]) -> int:
|
||||
return max(1, min(_DECODE_SPLIT_TAIL_VALUE, _cu_count() // T))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Split plan from aiter's v4 nm latency model (ROCm/aiter#6126, ATOM#2462)
|
||||
# ---------------------------------------------------------------------------
|
||||
# t(i) ~= L + W(i) * (C + A * kv_len / i) + [i > 1] * (S + D * num_seqs * i),
|
||||
# W(i) = ceil(num_seqs * ceil(H / 64) * i / CU). The gfx950 fit is aiter's
|
||||
# (gqa=128, qSeqLen=1, graph replay, uniform KV). Used only when the installed
|
||||
# aiter predates get_mla_v4_nm_num_kv_splits.
|
||||
_V4_NM_SPLIT_COST = {"gfx950": (12.74, 3.21, 0.0300, 3.38, 0.076)}
|
||||
V4_NM_MAX_SPLITS = 16
|
||||
# rows the constant split table covers; the asm wraps byte offsets past 32768
|
||||
V4_NM_SPLIT_TABLE_ROWS = 32768
|
||||
|
||||
|
||||
def _ported_num_kv_splits(
|
||||
num_seqs: int, num_heads: int, kv_len: int, gfx: str, cu_num: int
|
||||
) -> Optional[int]:
|
||||
cost = _V4_NM_SPLIT_COST.get(gfx)
|
||||
if cost is None:
|
||||
return None
|
||||
L, C, A, S, D = cost
|
||||
tg_factor = max(1, -(-num_heads // 64))
|
||||
kv_len = max(1, kv_len)
|
||||
best_t, best_s = None, 1
|
||||
for s in range(1, V4_NM_MAX_SPLITS + 1):
|
||||
waves = -(-num_seqs * tg_factor * s // cu_num)
|
||||
t = L + waves * (C + A * kv_len / s)
|
||||
if s > 1:
|
||||
t += S + D * num_seqs * s
|
||||
if best_t is None or t < best_t:
|
||||
best_t, best_s = t, s
|
||||
return best_s
|
||||
|
||||
|
||||
@lru_cache(maxsize=None)
|
||||
def fp8_decode_num_kv_splits(
|
||||
num_seqs: int, num_heads: int, kv_len: int
|
||||
) -> Optional[int]:
|
||||
"""aiter's latency-model split count for ``mla_decode_fwd_v4_nm``, or None
|
||||
(no model for this arch: keep the existing pick). Host only, cached: the
|
||||
inputs are fixed per captured graph, so capture and replay agree."""
|
||||
try:
|
||||
from aiter.mla import get_mla_v4_nm_num_kv_splits
|
||||
except ImportError:
|
||||
get_mla_v4_nm_num_kv_splits = None
|
||||
if get_mla_v4_nm_num_kv_splits is not None:
|
||||
return int(get_mla_v4_nm_num_kv_splits(num_seqs, num_heads, kv_len))
|
||||
from aiter.jit.utils.chip_info import get_gfx
|
||||
|
||||
return _ported_num_kv_splits(num_seqs, num_heads, kv_len, get_gfx(), _cu_count())
|
||||
|
||||
|
||||
_split_tables: dict = {}
|
||||
|
||||
|
||||
def fp8_decode_split_table(device: torch.device) -> Optional[torch.Tensor]:
|
||||
"""``[V4_NM_MAX_SPLITS, V4_NM_SPLIT_TABLE_ROWS + 1]`` int32, row ``s - 1``
|
||||
the uniform ``split_indptr`` ``[0, s, 2s, ...]``. Built once per device
|
||||
outside capture and never written again, so every plan is a slice of
|
||||
constant memory (no per-layer arange). None if it would be built under
|
||||
capture: that graph keeps the wrapper's own pick."""
|
||||
device = torch.device(device)
|
||||
key = device.index if device.index is not None else torch.cuda.current_device()
|
||||
table = _split_tables.get(key)
|
||||
if table is None:
|
||||
if torch.cuda.is_current_stream_capturing():
|
||||
return None
|
||||
splits = torch.arange(1, V4_NM_MAX_SPLITS + 1, dtype=torch.int32, device=device)
|
||||
table = splits[:, None] * torch.arange(
|
||||
V4_NM_SPLIT_TABLE_ROWS + 1, dtype=torch.int32, device=device
|
||||
)
|
||||
_split_tables[key] = table
|
||||
return table
|
||||
|
||||
|
||||
def fp8_decode_split_plan(
|
||||
n_seq: int, num_heads: int, kv_len_hint: int, device: torch.device
|
||||
) -> Optional[Tuple[int, torch.Tensor]]:
|
||||
"""(num_kv_splits, split_indptr) from the latency model, or None."""
|
||||
if n_seq > V4_NM_SPLIT_TABLE_ROWS:
|
||||
return None
|
||||
s = fp8_decode_num_kv_splits(n_seq, num_heads, int(kv_len_hint))
|
||||
if s is None:
|
||||
return None
|
||||
table = fp8_decode_split_table(device)
|
||||
if table is None:
|
||||
return None
|
||||
return s, table[s - 1, : n_seq + 1]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# HCA persistent decode (aiter mla_decode_fwd_v4_nm_ps, ROCm/aiter#6126)
|
||||
# ---------------------------------------------------------------------------
|
||||
HCA_PS_HEADS = 128
|
||||
HCA_PS_MAX_ROWS = 32768
|
||||
HCA_PS_PARTITIONS = 128
|
||||
_hca_ps_workspaces: dict = {}
|
||||
|
||||
|
||||
def _hca_ps_api():
|
||||
try:
|
||||
from aiter.mla import get_mla_v4_nm_ps_workspace, mla_decode_fwd_v4_nm_ps
|
||||
except ImportError:
|
||||
return None
|
||||
return get_mla_v4_nm_ps_workspace, mla_decode_fwd_v4_nm_ps
|
||||
|
||||
|
||||
def _dev_key(device) -> int:
|
||||
device = torch.device(device)
|
||||
return device.index if device.index is not None else torch.cuda.current_device()
|
||||
|
||||
|
||||
def prepare_fp8_hca_persist(device, num_heads: int) -> bool:
|
||||
"""Allocate this device's persistent-decode workspace (~64 MiB) when the
|
||||
switch is on and the kernel can serve this rank. Call outside capture,
|
||||
before warmup; idempotent. One workspace per device is enough because
|
||||
every decode layer runs on the forward's compute stream in order."""
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.utils import is_gfx95_supported
|
||||
|
||||
if not envs.SGLANG_DSV4_FP8_HCA_PERSIST.get() or num_heads != HCA_PS_HEADS:
|
||||
return False
|
||||
if not is_gfx95_supported():
|
||||
return False
|
||||
key = _dev_key(device)
|
||||
if key in _hca_ps_workspaces:
|
||||
return True
|
||||
api = _hca_ps_api()
|
||||
if api is None or torch.cuda.is_current_stream_capturing():
|
||||
return False
|
||||
_hca_ps_workspaces[key] = api[0](
|
||||
torch.device("cuda", key), num_partitions=HCA_PS_PARTITIONS
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
def fp8_hca_persist_wanted(
|
||||
*,
|
||||
T: int,
|
||||
H: int,
|
||||
v_head_dim: int,
|
||||
compress_ratio: Optional[int],
|
||||
max_seqlen_q: int,
|
||||
per_token: bool,
|
||||
device,
|
||||
) -> bool:
|
||||
from sglang.srt.environ import envs
|
||||
|
||||
if not (
|
||||
compress_ratio == 128
|
||||
and H == HCA_PS_HEADS
|
||||
and v_head_dim == 512
|
||||
and max_seqlen_q == 1
|
||||
and per_token
|
||||
and envs.SGLANG_DSV4_FP8_HCA_PERSIST.get()
|
||||
and envs.SGLANG_DSV4_FP8_HCA_PERSIST_MIN_ROWS.get() <= T <= HCA_PS_MAX_ROWS
|
||||
):
|
||||
return False
|
||||
# eager calls may allocate it late; a graph captured without one keeps asm
|
||||
return prepare_fp8_hca_persist(device, H)
|
||||
|
||||
|
||||
def decode_fp8_hca_ps(
|
||||
*,
|
||||
q: torch.Tensor, # [T, 128, 512] fp8 packed
|
||||
q_rope: torch.Tensor, # [T, 128, 64] bf16
|
||||
unified_kv: torch.Tensor,
|
||||
unified_kv_rope: torch.Tensor,
|
||||
kv_indices: torch.Tensor,
|
||||
kv_indptr: torch.Tensor,
|
||||
attn_sink: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""One-launch HCA decode: the kernel plans the split from ``kv_indptr`` on
|
||||
the GPU and merges the partials itself. Rows with an empty KV range are
|
||||
left unwritten (the builders never emit one; see decode_fp8_2buff)."""
|
||||
T = q.shape[0]
|
||||
out = q_rope.new_empty((T, HCA_PS_HEADS, 512))
|
||||
if T == 0:
|
||||
return out
|
||||
_hca_ps_api()[1](
|
||||
q,
|
||||
q_rope,
|
||||
unified_kv,
|
||||
unified_kv_rope,
|
||||
kv_indptr[: T + 1],
|
||||
kv_indices,
|
||||
attn_sink,
|
||||
_hca_ps_workspaces[_dev_key(q.device)],
|
||||
out=out,
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
# TODO(aiter#5882): remove once ROCm/aiter#5882 is in the aiter pin
|
||||
# (with it: the in-tree fallback below, flydsl_v4nm/ and
|
||||
# SGLANG_DSV4_FLYDSL_DECODE_SOURCE; aiter's copy is then the only one).
|
||||
_FLYDSL_V4_DECODE_SOURCES = ("auto", "aiter", "intree")
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def _flydsl_v4_decode():
|
||||
"""The FlyDSL v4 nm decode API (ROCm/aiter#5882) or None (asm kernel).
|
||||
|
||||
SGLANG_DSV4_FLYDSL_DECODE=0 forces None. SGLANG_DSV4_FLYDSL_DECODE_SOURCE:
|
||||
auto (default) takes aiter.ops.flydsl when the installed aiter ships it, else
|
||||
sglang's in-tree copy (flydsl_v4nm); aiter / intree take only that one.
|
||||
"""
|
||||
from sglang.srt.environ import envs
|
||||
|
||||
if not envs.SGLANG_DSV4_FLYDSL_DECODE.get():
|
||||
return None
|
||||
source = envs.SGLANG_DSV4_FLYDSL_DECODE_SOURCE.get()
|
||||
if source not in _FLYDSL_V4_DECODE_SOURCES:
|
||||
raise ValueError(
|
||||
f"SGLANG_DSV4_FLYDSL_DECODE_SOURCE={source!r}, expected one of "
|
||||
f"{_FLYDSL_V4_DECODE_SOURCES}"
|
||||
)
|
||||
fly = None
|
||||
if source != "intree":
|
||||
try:
|
||||
from aiter.ops import flydsl as fly
|
||||
|
||||
fly.flydsl_mla_v4_decode_supported # resolves the lazy import
|
||||
origin = "aiter.ops.flydsl"
|
||||
except (ImportError, AttributeError):
|
||||
fly = None
|
||||
if fly is None and source != "aiter":
|
||||
try:
|
||||
from sglang.kernels.ops.attention.dsv4 import flydsl_v4nm as fly
|
||||
|
||||
origin = "sglang in-tree copy of ROCm/aiter#5882"
|
||||
except ImportError as e: # no flydsl, or an aiter before ROCm/aiter#6377
|
||||
fly = None
|
||||
logger.warning("in-tree FlyDSL v4 nm decode unavailable: %s", e)
|
||||
if fly is None:
|
||||
logger.info("DSv4 fp8 decode: no FlyDSL v4 nm decode (source=%s)", source)
|
||||
else:
|
||||
logger.info("DSv4 fp8 decode: FlyDSL v4 nm decode from %s", origin)
|
||||
return fly
|
||||
|
||||
|
||||
def flydsl_decode_supported(
|
||||
num_heads: int, max_seqlen_q: int = 1, grouped: bool = False
|
||||
) -> bool:
|
||||
fly = _flydsl_v4_decode()
|
||||
if fly is None:
|
||||
return False
|
||||
if num_heads == 128:
|
||||
from sglang.srt.environ import envs
|
||||
|
||||
if not envs.SGLANG_DSV4_FLYDSL_DECODE_H128.get():
|
||||
return False
|
||||
return fly.flydsl_mla_v4_decode_supported(num_heads, max_seqlen_q, grouped)
|
||||
|
||||
|
||||
def _stream_hint(compress_ratio: Optional[int]) -> Optional[str]:
|
||||
# FlyDSL split plans are keyed by stream kind
|
||||
if compress_ratio is None:
|
||||
return None
|
||||
return {0: "swa", 4: "csa", 128: "hca"}[compress_ratio]
|
||||
|
||||
|
||||
def flydsl_grouped_verify_pays(
|
||||
num_heads: int, num_draft: int, num_reqs: int, compress_ratio: int, epi: str
|
||||
) -> bool:
|
||||
"""aiter's measured policy: whether target verify should read one shared
|
||||
stream per request on this layer kind, for output mode ``epi``."""
|
||||
return flydsl_decode_supported(
|
||||
num_heads, num_draft, grouped=True
|
||||
) and _flydsl_v4_decode().flydsl_mla_v4_grouped_pays(
|
||||
num_heads, num_draft, num_reqs, hint=_stream_hint(compress_ratio), epi=epi
|
||||
)
|
||||
|
||||
|
||||
def build_grouped_verify_streams(
|
||||
state_slot: torch.Tensor, positions: torch.Tensor, *args, **kwargs
|
||||
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""(kv_indices, kv_indptr, qo_indptr, q_kv_bounds): one stream per request,
|
||||
with per-draft key ranges equal to each draft's per-token stream."""
|
||||
return _flydsl_v4_decode().flydsl_mla_v4_grouped_verify_meta(
|
||||
state_slot.contiguous(), positions.contiguous(), *args, **kwargs
|
||||
)
|
||||
|
||||
|
||||
def decode_fp8_2buff(
|
||||
*,
|
||||
q: torch.Tensor, # [T, H, nope_row_bytes] fp8 packed nope + inline e8m0 scale
|
||||
@@ -296,8 +582,17 @@ def decode_fp8_2buff(
|
||||
max_seqlen_q: int = 1,
|
||||
num_kv_splits: Optional[int] = None,
|
||||
compress_ratio: Optional[int] = None,
|
||||
q_kv_bounds: Optional[torch.Tensor] = None,
|
||||
kv_len_hint: Optional[int] = None,
|
||||
) -> torch.Tensor:
|
||||
"""Decode over the two-pool fp8 unified_kv, through aiter's v4 nm asm kernel.
|
||||
"""Decode over the two-pool fp8 unified_kv, through aiter's FlyDSL v4 nm
|
||||
decode (split merge in the same launch) where available, else the asm kernel.
|
||||
|
||||
On the asm kernel, with ``SGLANG_DSV4_FP8_HCA_PERSIST`` the HCA stream
|
||||
(H128, one q row per sequence) takes the persistent kernel, and with
|
||||
``SGLANG_DSV4_FP8_DECODE_SPLIT_MODEL`` plus ``kv_len_hint`` (the longest
|
||||
KV a row of this stream can see, fixed per server) the split count comes
|
||||
from aiter's latency model with a constant ``split_indptr``.
|
||||
|
||||
Q arrives in the same packed form as the pool rows (nope fp8 + duplicated
|
||||
e8m0 tile scales) with its rope half beside it in bf16, which is why this
|
||||
@@ -314,6 +609,9 @@ def decode_fp8_2buff(
|
||||
|
||||
``max_seqlen_q`` above 1 puts several q rows in one tile, which the v4
|
||||
dispatcher serves for gqa=16 at 1, 2 and 4 off the same 64-row tile.
|
||||
|
||||
``q_kv_bounds`` (FlyDSL only): the grouped verify streams of
|
||||
``build_grouped_verify_streams``.
|
||||
"""
|
||||
from aiter.mla import mla_decode_fwd_v4_nm
|
||||
|
||||
@@ -350,15 +648,52 @@ def decode_fp8_2buff(
|
||||
|
||||
rows = unified_kv.shape[0]
|
||||
out = q_rope.new_empty((n_seq * max_seqlen_q, H, v_head_dim))
|
||||
# Left None, the wrapper's occupancy heuristic picks it, folds the cross-split
|
||||
# merge back into `out`, and leaves the final bf16 there whether or not it
|
||||
# split. Pinning it to 1 costs 6.9x at bs=1 kv=2048.
|
||||
# Sequences, not tokens: the wave-fitting rule counts the workgroups the
|
||||
# kernel launches, which is n_seq x splits. They are the same number on the
|
||||
# per-token path and differ once q rows share a tile.
|
||||
if num_kv_splits is None and n_seq > _DECODE_SPLIT_TAIL_MIN_TOKENS:
|
||||
num_kv_splits = _decode_fp8_tail_splits(n_seq, compress_ratio)
|
||||
mla_decode_fwd_v4_nm(
|
||||
# FlyDSL is a drop-in for the asm kernel: its own capture-safe split plan,
|
||||
# split merge in the same launch
|
||||
if flydsl_decode_supported(H, max_seqlen_q, q_kv_bounds is not None):
|
||||
decode_fwd = _flydsl_v4_decode().flydsl_mla_decode_fwd_v4_nm
|
||||
kwargs = dict(hint=_stream_hint(compress_ratio), q_kv_bounds=q_kv_bounds)
|
||||
else:
|
||||
assert q_kv_bounds is None, "grouped verify streams need FlyDSL"
|
||||
if fp8_hca_persist_wanted(
|
||||
T=T,
|
||||
H=H,
|
||||
v_head_dim=v_head_dim,
|
||||
compress_ratio=compress_ratio,
|
||||
max_seqlen_q=max_seqlen_q,
|
||||
per_token=n_seq == T,
|
||||
device=q.device,
|
||||
):
|
||||
return decode_fp8_hca_ps(
|
||||
q=q,
|
||||
q_rope=q_rope,
|
||||
unified_kv=unified_kv,
|
||||
unified_kv_rope=unified_kv_rope,
|
||||
kv_indices=kv_indices,
|
||||
kv_indptr=kv_indptr,
|
||||
attn_sink=attn_sink,
|
||||
)
|
||||
split_indptr = None
|
||||
if num_kv_splits is None and kv_len_hint is not None:
|
||||
from sglang.srt.environ import envs
|
||||
|
||||
if envs.SGLANG_DSV4_FP8_DECODE_SPLIT_MODEL.get():
|
||||
plan = fp8_decode_split_plan(n_seq, H, kv_len_hint, q.device)
|
||||
if plan is not None:
|
||||
num_kv_splits, split_indptr = plan
|
||||
# Left None, the wrapper's occupancy heuristic picks it, folds the
|
||||
# cross-split merge back into `out`, and leaves the final bf16 there
|
||||
# whether or not it split. Pinning it to 1 costs 6.9x at bs=1 kv=2048.
|
||||
# Sequences, not tokens: the wave-fitting rule counts the workgroups the
|
||||
# kernel launches, which is n_seq x splits. They are the same number on
|
||||
# the per-token path and differ once q rows share a tile.
|
||||
if num_kv_splits is None and n_seq > _DECODE_SPLIT_TAIL_MIN_TOKENS:
|
||||
num_kv_splits = _decode_fp8_tail_splits(n_seq, compress_ratio)
|
||||
decode_fwd, kwargs = (
|
||||
mla_decode_fwd_v4_nm,
|
||||
dict(num_kv_splits=num_kv_splits, split_indptr=split_indptr),
|
||||
)
|
||||
decode_fwd(
|
||||
q,
|
||||
q_rope,
|
||||
unified_kv.view(rows, 1, 1, row_bytes),
|
||||
@@ -369,7 +704,7 @@ def decode_fp8_2buff(
|
||||
kv_indices,
|
||||
max_seqlen_q,
|
||||
sink=attn_sink,
|
||||
num_kv_splits=num_kv_splits,
|
||||
**kwargs,
|
||||
)
|
||||
# No empty-segment mask: a CG-padded row gets seq_len 1 on the ring slot
|
||||
# ReqToTokenPool reserves, so the builders can't emit a zero-length one, and
|
||||
@@ -378,6 +713,41 @@ def decode_fp8_2buff(
|
||||
return out[:T]
|
||||
|
||||
|
||||
def decode_fp8_2buff_wo_a_mxfp8(
|
||||
*,
|
||||
q: torch.Tensor, # [T, H, nope_row_bytes] fp8 packed nope + inline e8m0 scale
|
||||
q_rope: torch.Tensor, # [T, H, rope_dim] bf16
|
||||
unified_kv: torch.Tensor, # [rows, nope_row_bytes] fp8
|
||||
unified_kv_rope: torch.Tensor, # [rows, rope_dim] bf16
|
||||
kv_indices: torch.Tensor,
|
||||
kv_indptr: torch.Tensor,
|
||||
attn_sink: torch.Tensor, # [H] fp32
|
||||
positions: torch.Tensor, # [T] int64
|
||||
rope_freqs: torch.Tensor, # [max_pos, rope_dim] fp32, view_as_real(freqs_cis)
|
||||
compress_ratio: int,
|
||||
qo_indptr: Optional[torch.Tensor] = None,
|
||||
max_seqlen_q: int = 1,
|
||||
q_kv_bounds: Optional[torch.Tensor] = None,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
"""``decode_fp8_2buff`` + inverse RoPE + wo_a mxfp8 quant in one FlyDSL
|
||||
launch. Returns the wo_a operands (fp8 [T, H*512], e8m0 [T, H*4])."""
|
||||
return _flydsl_v4_decode().flydsl_mla_v4_decode_fused(
|
||||
q,
|
||||
q_rope,
|
||||
unified_kv,
|
||||
unified_kv_rope,
|
||||
kv_indptr,
|
||||
kv_indices,
|
||||
attn_sink,
|
||||
positions,
|
||||
rope_freqs,
|
||||
compress_ratio,
|
||||
qo_indptr=qo_indptr,
|
||||
max_seqlen_q=max_seqlen_q,
|
||||
q_kv_bounds=q_kv_bounds,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _fill_compress_tail_kernel(
|
||||
indices_ptr, # [*] int32 (out)
|
||||
|
||||
@@ -1613,6 +1613,31 @@ class Envs:
|
||||
# only way to ask; on separate-KV it is the reverse -- --kv-cache-dtype
|
||||
# picks the buffer dtype and this switch is inert.
|
||||
SGLANG_DSV4_UNIFIED_KV_FP8 = EnvBool(False)
|
||||
# fp8 unified_kv decode/verify on aiter's FlyDSL v4 nm decode when the
|
||||
# installed aiter ships it; 0 keeps the asm kernel.
|
||||
SGLANG_DSV4_FLYDSL_DECODE = EnvBool(True)
|
||||
# ... which copy: "auto" = aiter's when the installed aiter ships it, else
|
||||
# sglang's in-tree copy; "aiter" = aiter's only (asm without it); "intree" =
|
||||
# the in-tree copy only.
|
||||
# TODO(aiter#5882): remove once ROCm/aiter#5882 is in the aiter pin
|
||||
SGLANG_DSV4_FLYDSL_DECODE_SOURCE = EnvStr("auto")
|
||||
# ... also with 128 local heads (DP attention). Off: there the asm path
|
||||
# below (split plan + HCA persistent, ROCm/aiter#6126) measured the same
|
||||
# DP8 DSpark decode step on MI355X without depending on FlyDSL.
|
||||
SGLANG_DSV4_FLYDSL_DECODE_H128 = EnvBool(False)
|
||||
# fp8 unified_kv asm decode (when FlyDSL is off or absent): pick the KV
|
||||
# split count from aiter's v4 nm latency model (ROCm/aiter#6126; ported
|
||||
# when aiter predates it) with a constant split_indptr, per stream kind.
|
||||
SGLANG_DSV4_FP8_DECODE_SPLIT_MODEL = EnvBool(True)
|
||||
# context the HCA split plan is sized for, capped at the server context
|
||||
# length (0: the server context length). Sizing for 1M over-splits HCA at
|
||||
# AgentX lengths (32k-200k).
|
||||
SGLANG_DSV4_FP8_DECODE_SPLIT_CTX = EnvInt(200_000)
|
||||
# fp8 unified_kv asm decode: HCA layers with 128 local heads (DP attention)
|
||||
# take aiter's persistent mla_decode_fwd_v4_nm_ps (in-kernel split + merge)
|
||||
# for MIN_ROWS..32768 rows; needs aiter >= ROCm/aiter#6126.
|
||||
SGLANG_DSV4_FP8_HCA_PERSIST = EnvBool(True)
|
||||
SGLANG_DSV4_FP8_HCA_PERSIST_MIN_ROWS = EnvInt(15)
|
||||
|
||||
# DeepSeek-V4.1 engram host table: keep the tables in host memory (layout
|
||||
# below) and gather rows from the GPU instead of sharding them over HBM.
|
||||
|
||||
@@ -270,6 +270,16 @@ class UnifiedKvMetadata:
|
||||
gasm_kv_indptr: Optional[torch.Tensor] = None
|
||||
gasm_qo_indptr: Optional[torch.Tensor] = None
|
||||
|
||||
# Grouped target-verify streams for the FlyDSL fp8 decode: one per request
|
||||
# on HCA / SWA-only layers, with per-draft key ranges (q_kv_bounds).
|
||||
grp_qo_indptr: Optional[torch.Tensor] = None
|
||||
grp_hca_indices: Optional[torch.Tensor] = None
|
||||
grp_hca_indptr: Optional[torch.Tensor] = None
|
||||
grp_hca_bounds: Optional[torch.Tensor] = None
|
||||
grp_swa_indices: Optional[torch.Tensor] = None
|
||||
grp_swa_indptr: Optional[torch.Tensor] = None
|
||||
grp_swa_bounds: Optional[torch.Tensor] = None
|
||||
|
||||
# prefill/extend per-token mapping
|
||||
pf_state_slot: Optional[torch.Tensor] = None
|
||||
pf_chunk_start: Optional[torch.Tensor] = None
|
||||
@@ -298,6 +308,13 @@ class UnifiedKvMetadata:
|
||||
"gasm_indices",
|
||||
"gasm_kv_indptr",
|
||||
"gasm_qo_indptr",
|
||||
"grp_qo_indptr",
|
||||
"grp_hca_indices",
|
||||
"grp_hca_indptr",
|
||||
"grp_hca_bounds",
|
||||
"grp_swa_indices",
|
||||
"grp_swa_indptr",
|
||||
"grp_swa_bounds",
|
||||
"csa_indices",
|
||||
"csa_indptr",
|
||||
"pf_state_slot",
|
||||
@@ -330,6 +347,13 @@ class UnifiedKvMetadata:
|
||||
"gasm_indices",
|
||||
"gasm_kv_indptr",
|
||||
"gasm_qo_indptr",
|
||||
"grp_qo_indptr",
|
||||
"grp_hca_indices",
|
||||
"grp_hca_indptr",
|
||||
"grp_hca_bounds",
|
||||
"grp_swa_indices",
|
||||
"grp_swa_indptr",
|
||||
"grp_swa_bounds",
|
||||
"csa_indices",
|
||||
"csa_indptr",
|
||||
"pf_state_slot",
|
||||
@@ -971,6 +995,17 @@ class DeepseekV4HipRadixBackend(
|
||||
# DSpark draft workers verify gamma rows; the server arg keeps the
|
||||
# CUDA-side convention gamma + 1.
|
||||
self.target_verify_num_draft_tokens = self.speculative_num_draft_tokens - 1
|
||||
self._fp8_decode_num_heads = (
|
||||
model_runner.model_config.num_attention_heads // get_parallel().attn_tp_size
|
||||
)
|
||||
# output mode of the fp8 verify: deepseek_v4 fuses the fp8 wo_a quant into it
|
||||
self._fp8_verify_epi = (
|
||||
"invrope_mxfp8"
|
||||
if any(getattr(m, "wo_a_fp8", False) for m in model_runner.model.modules())
|
||||
else "bf16"
|
||||
)
|
||||
self._fp8_split_kv_hint: Dict[int, int] = {}
|
||||
self._init_fp8_asm_decode_plan()
|
||||
# Past MAX_FUSED_ROWS the fp4 schedule falls back to AITER's preamble,
|
||||
# which frees the scratch its kernels read -- not capture-safe.
|
||||
self._fp4_graph_row_limit: Optional[int] = None
|
||||
@@ -2409,6 +2444,37 @@ class DeepseekV4HipRadixBackend(
|
||||
if current_raw is not None:
|
||||
self.forward_metadata = current_raw
|
||||
|
||||
def _init_fp8_asm_decode_plan(self) -> None:
|
||||
"""fp8 unified_kv asm decode: per-stream KV bound for the split plan
|
||||
and, at init (outside capture, before warmup), the constant split
|
||||
table and the HCA persistent workspace."""
|
||||
from sglang.kernels.ops.attention.dsv4.unified_kv_kernels.env_gate import (
|
||||
is_unified_kv_fp8,
|
||||
)
|
||||
|
||||
if self.is_draft_worker or not is_unified_kv_fp8():
|
||||
return
|
||||
from sglang.kernels.ops.attention.dsv4.unified_kv_kernels import runtime
|
||||
|
||||
win = self.token_to_kv_pool.unified_swa_window
|
||||
ctx = min(
|
||||
envs.SGLANG_DSV4_FP8_DECODE_SPLIT_CTX.get() or self.max_context_len,
|
||||
self.max_context_len,
|
||||
)
|
||||
# longest KV one decode row of each stream can see
|
||||
self._fp8_split_kv_hint = {
|
||||
0: win,
|
||||
4: win + self.index_topk,
|
||||
128: win + ctx // 128,
|
||||
}
|
||||
if envs.SGLANG_DSV4_FP8_DECODE_SPLIT_MODEL.get():
|
||||
runtime.fp8_decode_split_table(self.device)
|
||||
if runtime.prepare_fp8_hca_persist(self.device, self._fp8_decode_num_heads):
|
||||
logger.info(
|
||||
"DSV4 fp8 decode: HCA layers on the persistent asm kernel "
|
||||
f"(rows >= {envs.SGLANG_DSV4_FP8_HCA_PERSIST_MIN_ROWS.get()})"
|
||||
)
|
||||
|
||||
def _attach_unified_kv_decode_streams(
|
||||
self,
|
||||
core: DSV4AttnMetadata,
|
||||
@@ -2418,6 +2484,7 @@ class DeepseekV4HipRadixBackend(
|
||||
# state_slot maps each query token to its request slot;
|
||||
# target-verify repeats request slots for the draft tokens.
|
||||
from sglang.kernels.ops.attention.dsv4.unified_kv_kernels.env_gate import (
|
||||
is_unified_kv_fp8,
|
||||
is_unified_kv_triton,
|
||||
)
|
||||
|
||||
@@ -2493,6 +2560,43 @@ class DeepseekV4HipRadixBackend(
|
||||
block_q=_GROUPED_ASM_BLOCK_Q,
|
||||
)
|
||||
|
||||
u = core.unified
|
||||
u.grp_qo_indptr = None
|
||||
u.grp_hca_indices = u.grp_hca_indptr = u.grp_hca_bounds = None
|
||||
u.grp_swa_indices = u.grp_swa_indptr = u.grp_swa_bounds = None
|
||||
num_reqs = N // num_draft if num_draft > 1 and N % num_draft == 0 else 0
|
||||
if self.is_dspark and num_reqs and is_unified_kv_fp8():
|
||||
grouped = functools.partial(
|
||||
runtime.build_grouped_verify_streams,
|
||||
state_slot,
|
||||
core.positions_casual,
|
||||
num_draft=num_draft,
|
||||
win=pool.unified_swa_window,
|
||||
ring_stride=pool.unified_swa_ring_size,
|
||||
swa_pages=pool.unified_swa_pages,
|
||||
)
|
||||
pays = functools.partial(
|
||||
runtime.flydsl_grouped_verify_pays,
|
||||
self._fp8_decode_num_heads,
|
||||
num_draft,
|
||||
num_reqs,
|
||||
epi=self._fp8_verify_epi,
|
||||
)
|
||||
if pays(128):
|
||||
(
|
||||
u.grp_hca_indices,
|
||||
u.grp_hca_indptr,
|
||||
u.grp_qo_indptr,
|
||||
u.grp_hca_bounds,
|
||||
) = grouped(hca_len, hca_page_indices)
|
||||
if pays(0):
|
||||
(
|
||||
u.grp_swa_indices,
|
||||
u.grp_swa_indptr,
|
||||
u.grp_qo_indptr,
|
||||
u.grp_swa_bounds,
|
||||
) = grouped(None, None)
|
||||
|
||||
# SWA ring write target, same value for every layer this forward.
|
||||
req_slot = state_slot.to(torch.int64)
|
||||
core.unified.swa_loc = (
|
||||
@@ -2574,9 +2678,12 @@ class DeepseekV4HipRadixBackend(
|
||||
save_kv_cache: bool = True,
|
||||
q_rope: Optional[torch.Tensor] = None,
|
||||
k_rope: Optional[torch.Tensor] = None,
|
||||
) -> torch.Tensor:
|
||||
wo_a_quant_freqs: Optional[torch.Tensor] = None,
|
||||
) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]:
|
||||
"""q_rope present: packed fp8 q over the two-pool fp8 layout (asm decode;
|
||||
prefill also needs k_rope). Absent: plain bf16 q and pool (Triton)."""
|
||||
prefill also needs k_rope). Absent: plain bf16 q and pool (Triton).
|
||||
wo_a_quant_freqs (fp8 decode): return the inverse-RoPE'd wo_a mxfp8
|
||||
operands instead of the attention output."""
|
||||
from sglang.kernels.ops.attention.dsv4.unified_kv_kernels import runtime
|
||||
|
||||
pool = self.token_to_kv_pool
|
||||
@@ -2650,9 +2757,44 @@ class DeepseekV4HipRadixBackend(
|
||||
"the v4 nm asm kernel hardcodes 1/sqrt(512), this backend is "
|
||||
f"at {self.softmax_scale}"
|
||||
)
|
||||
qo_indptr, max_seqlen_q, q_kv_bounds = None, 1, None
|
||||
grp = {
|
||||
128: (
|
||||
unified_metadata.grp_hca_indices,
|
||||
unified_metadata.grp_hca_indptr,
|
||||
unified_metadata.grp_hca_bounds,
|
||||
),
|
||||
0: (
|
||||
unified_metadata.grp_swa_indices,
|
||||
unified_metadata.grp_swa_indptr,
|
||||
unified_metadata.grp_swa_bounds,
|
||||
),
|
||||
}.get(compress_ratio)
|
||||
if verify_as_decode and grp is not None and grp[0] is not None:
|
||||
kv_indices, kv_indptr, q_kv_bounds = grp
|
||||
qo_indptr = unified_metadata.grp_qo_indptr
|
||||
max_seqlen_q = self.target_verify_num_draft_tokens
|
||||
if wo_a_quant_freqs is not None:
|
||||
return runtime.decode_fp8_2buff_wo_a_mxfp8(
|
||||
q=q,
|
||||
q_rope=q_rope,
|
||||
unified_kv=unified,
|
||||
unified_kv_rope=pool.get_unified_kv_rope(layer_id),
|
||||
kv_indices=kv_indices,
|
||||
kv_indptr=kv_indptr,
|
||||
attn_sink=attn_sink,
|
||||
positions=positions,
|
||||
rope_freqs=wo_a_quant_freqs,
|
||||
compress_ratio=compress_ratio,
|
||||
qo_indptr=qo_indptr,
|
||||
max_seqlen_q=max_seqlen_q,
|
||||
q_kv_bounds=q_kv_bounds,
|
||||
)
|
||||
gasm = (
|
||||
unified_metadata.gasm_indices
|
||||
if compress_ratio == 128 and q.shape[1] == _GROUPED_ASM_GQA
|
||||
if compress_ratio == 128
|
||||
and q.shape[1] == _GROUPED_ASM_GQA
|
||||
and q_kv_bounds is None
|
||||
else None
|
||||
)
|
||||
if gasm is not None:
|
||||
@@ -2679,6 +2821,14 @@ class DeepseekV4HipRadixBackend(
|
||||
attn_sink=attn_sink,
|
||||
v_head_dim=layer.v_head_dim,
|
||||
compress_ratio=compress_ratio,
|
||||
qo_indptr=qo_indptr,
|
||||
max_seqlen_q=max_seqlen_q,
|
||||
q_kv_bounds=q_kv_bounds,
|
||||
kv_len_hint=(
|
||||
self._fp8_split_kv_hint.get(compress_ratio)
|
||||
if qo_indptr is None
|
||||
else None
|
||||
),
|
||||
)
|
||||
from sglang.kernels.ops.attention.dsv4.unified_kv_kernels.paged_decode import (
|
||||
_kv_splits_for_stream,
|
||||
@@ -3034,8 +3184,9 @@ class DeepseekV4HipRadixBackend(
|
||||
q_rope: Optional[torch.Tensor] = None,
|
||||
k_rope: Optional[torch.Tensor] = None,
|
||||
inv_rope: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
|
||||
wo_a_quant_freqs: Optional[torch.Tensor] = None,
|
||||
**_,
|
||||
) -> torch.Tensor:
|
||||
) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]:
|
||||
"""inv_rope=(freqs_real, positions) has the kernel apply the inverse RoPE
|
||||
to the last freqs_real.shape[-1] dims of every output head."""
|
||||
if self.mtp_enabled and forward_batch.forward_mode.is_idle():
|
||||
@@ -3066,6 +3217,7 @@ class DeepseekV4HipRadixBackend(
|
||||
save_kv_cache=save_kv_cache,
|
||||
q_rope=q_rope,
|
||||
k_rope=k_rope,
|
||||
wo_a_quant_freqs=wo_a_quant_freqs,
|
||||
)
|
||||
assert inv_rope is None, (
|
||||
"the unified_kv model path applies its inverse RoPE"
|
||||
|
||||
@@ -230,6 +230,24 @@ def apply_wo_a_fp8_mxscale_fused_invrope(
|
||||
)
|
||||
|
||||
|
||||
def apply_wo_a_fp8_mxscale_prequant(
|
||||
o_fp8: torch.Tensor,
|
||||
o_scale: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
weight_scale: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""fp8 ``wo_a`` on ``quant_wo_a_act_mxfp8``-layout operands flattened per
|
||||
token (fp8 ``[T, G*D]``, e8m0 ``[T, G*D/128]``). Returns bf16 ``[T, G, R]``."""
|
||||
T, G = o_fp8.shape[0], weight.shape[0]
|
||||
return _batched_gemm_a8w8_mxscale(
|
||||
o_fp8.view(T, G, -1),
|
||||
weight,
|
||||
o_scale.view(T, G, -1),
|
||||
weight_scale,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
|
||||
|
||||
def _is_power_of_two(scale: torch.Tensor) -> bool:
|
||||
"""True when every fp32 scale is an exact power of two (zero mantissa)."""
|
||||
bits = scale.detach().float().contiguous().view(torch.int32)
|
||||
@@ -287,6 +305,7 @@ def wo_a_weight_scale_to_e8m0(
|
||||
__all__ = [
|
||||
"WO_A_MXFP8_GROUP_SIZE",
|
||||
"apply_wo_a_fp8_mxscale",
|
||||
"apply_wo_a_fp8_mxscale_prequant",
|
||||
"is_wo_a_fp8_mxscale_supported",
|
||||
"quant_wo_a_act_mxfp8",
|
||||
"wo_a_weight_scale_to_e8m0",
|
||||
|
||||
@@ -418,6 +418,7 @@ _wo_a_aiter_batched_gemm_disabled = False
|
||||
# ``None`` here means the platform keeps the bf16 absorb GEMM.
|
||||
_wo_a_fp8_mxscale = None
|
||||
_wo_a_fp8_mxscale_fused_invrope = None
|
||||
_wo_a_fp8_mxscale_prequant = None
|
||||
_wo_a_weight_scale_to_e8m0 = None
|
||||
_hip = None
|
||||
if _is_hip:
|
||||
@@ -425,6 +426,7 @@ if _is_hip:
|
||||
from sglang.srt.models.deepseek_common.amd.deepseek_v4_wo_a_fp8 import (
|
||||
apply_wo_a_fp8_mxscale,
|
||||
apply_wo_a_fp8_mxscale_fused_invrope,
|
||||
apply_wo_a_fp8_mxscale_prequant,
|
||||
is_wo_a_fp8_fused_invrope_supported,
|
||||
is_wo_a_fp8_mxscale_supported,
|
||||
wo_a_weight_scale_to_e8m0,
|
||||
@@ -440,6 +442,7 @@ if _is_hip:
|
||||
and is_wo_a_fp8_fused_invrope_supported()
|
||||
):
|
||||
_wo_a_fp8_mxscale_fused_invrope = apply_wo_a_fp8_mxscale_fused_invrope
|
||||
_wo_a_fp8_mxscale_prequant = apply_wo_a_fp8_mxscale_prequant
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=1)
|
||||
@@ -1077,6 +1080,27 @@ class MqaAttentionBase(nn.Module):
|
||||
# Other FlashMLA implementations retain their existing padded shape.
|
||||
return 64 if self.n_local_heads <= 64 else self.n_heads
|
||||
|
||||
def _wo_a_quant_freqs(
|
||||
self, forward_batch: ForwardBatch, q_rope: Optional[torch.Tensor]
|
||||
) -> Optional[torch.Tensor]:
|
||||
"""Rope table when the fp8 unified_kv decode (aiter FlyDSL) should also
|
||||
do the inverse RoPE and the wo_a mxfp8 quant; else None."""
|
||||
mode = forward_batch.forward_mode
|
||||
if q_rope is None or not (mode.is_decode() or mode.is_target_verify()):
|
||||
return None
|
||||
if not hasattr(self, "_wo_a_freqs"):
|
||||
from sglang.kernels.ops.attention.dsv4.unified_kv_kernels import runtime
|
||||
|
||||
fused = (
|
||||
_wo_a_fp8_mxscale_prequant is not None
|
||||
and self.wo_a_fp8
|
||||
and runtime.flydsl_decode_supported(q_rope.shape[1])
|
||||
)
|
||||
self._wo_a_freqs = (
|
||||
torch.view_as_real(self.freqs_cis).flatten(1) if fused else None
|
||||
)
|
||||
return self._wo_a_freqs
|
||||
|
||||
def _local_attn_sink(self, kernel_num_heads: Optional[int] = None) -> torch.Tensor:
|
||||
if self.attn_tp_size == 1:
|
||||
return self.attn_sink
|
||||
@@ -2559,6 +2583,7 @@ class MQALayer(MqaAttentionBase):
|
||||
if _is_hip
|
||||
else None
|
||||
)
|
||||
wo_a_quant_freqs = None
|
||||
if unified:
|
||||
# only the HIP radix backend takes these; passing them always would
|
||||
# leave non-ROCm depending on the **_ in its forward() to drop them, and
|
||||
@@ -2570,6 +2595,9 @@ class MQALayer(MqaAttentionBase):
|
||||
rope_kwargs["k_rope"] = k_rope
|
||||
if inv_rope is not None:
|
||||
rope_kwargs["inv_rope"] = inv_rope
|
||||
wo_a_quant_freqs = self._wo_a_quant_freqs(forward_batch, q_rope)
|
||||
if wo_a_quant_freqs is not None:
|
||||
rope_kwargs["wo_a_quant_freqs"] = wo_a_quant_freqs
|
||||
o = attn_backend.forward(
|
||||
q=q_out if q_out is not None else q,
|
||||
k=attn_k,
|
||||
@@ -2613,7 +2641,15 @@ class MQALayer(MqaAttentionBase):
|
||||
**({} if inv_rope is None else {"inv_rope": inv_rope}),
|
||||
)
|
||||
o = o[:, tp_slice, :]
|
||||
if (
|
||||
if wo_a_quant_freqs is not None:
|
||||
# the decode already returned the wo_a mxfp8 (codes, scales)
|
||||
G = self.n_local_groups
|
||||
o = _wo_a_fp8_mxscale_prequant(
|
||||
*o,
|
||||
self.wo_a.weight.view(G, self.o_lora_rank, -1),
|
||||
self.wo_a.weight_scale_inv.data,
|
||||
)
|
||||
elif (
|
||||
self.wo_a_fp8
|
||||
and _wo_a_fp8_mxscale_fused_invrope is not None
|
||||
and not _is_npu
|
||||
|
||||
Reference in New Issue
Block a user