From ab5858ac8a8f2eb4dac3835e22e1a3f1cf9f1504 Mon Sep 17 00:00:00 2001 From: kk <43161300+kkHuang-amd@users.noreply.github.com> Date: Fri, 16 Oct 2026 16:50:16 +0800 Subject: [PATCH] [Perf] [AMD] dsv4: fp8 unified_kv decode/verify on FlyDSL v4 nm decode with a latency-model split plan (1/8) (#44016) Co-authored-by: Duyi-Wang --- .pre-commit-config.yaml | 3 + .../sglang/kernels/ops/attention/__init__.py | 13 + .../attention/dsv4/flydsl_v4nm/__init__.py | 41 + .../dsv4/flydsl_v4nm/kernels/__init__.py | 2 + .../kernels/mla_v4_decode_common.py | 252 + .../kernels/mla_v4_decode_gfx950.py | 5271 +++++++++++++++++ .../dsv4/flydsl_v4nm/mla_v4_decode.py | 718 +++ .../dsv4/flydsl_v4nm/mla_v4_grouped.py | 174 + .../dsv4/unified_kv_kernels/runtime.py | 392 +- python/sglang/srt/environ.py | 25 + .../deepseek_v4_backend_hip_radix.py | 160 +- .../amd/deepseek_v4_wo_a_fp8.py | 19 + python/sglang/srt/models/deepseek_v4.py | 38 +- .../e2e/dsv4/test_dsv4_unified_fp8_decode.py | 367 +- .../attention/dsv4/test_flydsl_v4nm_decode.py | 552 ++ 15 files changed, 7994 insertions(+), 33 deletions(-) create mode 100644 python/sglang/kernels/ops/attention/dsv4/flydsl_v4nm/__init__.py create mode 100644 python/sglang/kernels/ops/attention/dsv4/flydsl_v4nm/kernels/__init__.py create mode 100644 python/sglang/kernels/ops/attention/dsv4/flydsl_v4nm/kernels/mla_v4_decode_common.py create mode 100644 python/sglang/kernels/ops/attention/dsv4/flydsl_v4nm/kernels/mla_v4_decode_gfx950.py create mode 100644 python/sglang/kernels/ops/attention/dsv4/flydsl_v4nm/mla_v4_decode.py create mode 100644 python/sglang/kernels/ops/attention/dsv4/flydsl_v4nm/mla_v4_grouped.py create mode 100644 test/registered/kernels/ops/attention/dsv4/test_flydsl_v4nm_decode.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f86911aef9..dca528f292 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -56,6 +56,9 @@ repos: hooks: - id: codespell args: ['--config', '.codespellrc'] + # Vendored ROCm/aiter#5882 kernel, kept byte-for-byte (`padd` is a helper name). + # TODO(aiter#5882): remove once ROCm/aiter#5882 is in the aiter pin + exclude: ^python/sglang/kernels/ops/attention/dsv4/flydsl_v4nm/ - repo: https://github.com/pre-commit/mirrors-clang-format rev: v20.1.7 hooks: diff --git a/python/sglang/kernels/ops/attention/__init__.py b/python/sglang/kernels/ops/attention/__init__.py index 588d290ce2..4d277dd438 100644 --- a/python/sglang/kernels/ops/attention/__init__.py +++ b/python/sglang/kernels/ops/attention/__init__.py @@ -277,6 +277,19 @@ for _mod, _fn, _backend, _device in [ KernelBackend.FLYDSL, CapabilityRequirement.HIP, ), + # TODO(aiter#5882): remove once ROCm/aiter#5882 is in the aiter pin + ( + "dsv4.flydsl_v4nm.mla_v4_decode", + "flydsl_mla_decode_fwd_v4_nm", + KernelBackend.FLYDSL, + CapabilityRequirement.HIP, + ), + ( + "dsv4.flydsl_v4nm.mla_v4_decode", + "flydsl_mla_v4_decode_fused", + KernelBackend.FLYDSL, + CapabilityRequirement.HIP, + ), ]: register_kernel( KernelSpec( diff --git a/python/sglang/kernels/ops/attention/dsv4/flydsl_v4nm/__init__.py b/python/sglang/kernels/ops/attention/dsv4/flydsl_v4nm/__init__.py new file mode 100644 index 0000000000..2306a40efa --- /dev/null +++ b/python/sglang/kernels/ops/attention/dsv4/flydsl_v4nm/__init__.py @@ -0,0 +1,41 @@ +# TODO(aiter#5882): remove once ROCm/aiter#5882 is in the aiter pin +"""In-tree copy of ROCm/aiter#5882: DeepSeek-V4 fp8 MLA decode (v4 nm contract) +in one FlyDSL launch on gfx950, for aiter pins that do not ship it yet. + +Vendored from ROCm/aiter#5882 at commit 03359c43df (the head adapted to +ROCm/aiter#6377); see the per-file headers. The entry points match the ones +the PR adds to ``aiter.ops.flydsl``. ``unified_kv_kernels.runtime`` prefers +aiter's own copy and only falls back to this one (SGLANG_DSV4_FLYDSL_DECODE_SOURCE). + +Needs an aiter with ROCm/aiter#6377 (typed ``_to_ptr_global``): importing this +package raises ImportError on an older aiter, so callers fall back to asm. +""" + +import inspect + +# AITER owns the FlyDSL toolchain bootstrap and the shared tensor/buffer shims. +# Import it before the local kernel modules so its FlyDSL setup is active. +import aiter as _aiter # noqa: F401 +from aiter.ops.flydsl.kernels import communication_ops_utils as _cu + +if "dtype" not in inspect.signature(_cu._to_ptr_global).parameters: + raise ImportError( + "the in-tree FlyDSL v4 nm decode (ROCm/aiter#5882) needs an aiter with " + "ROCm/aiter#6377 (typed communication_ops_utils._to_ptr_global)" + ) + +from .mla_v4_decode import ( # noqa: E402 + flydsl_mla_decode_fwd_v4_nm, + flydsl_mla_v4_decode_fused, + flydsl_mla_v4_decode_supported, + flydsl_mla_v4_grouped_pays, +) +from .mla_v4_grouped import flydsl_mla_v4_grouped_verify_meta # noqa: E402 + +__all__ = [ + "flydsl_mla_decode_fwd_v4_nm", + "flydsl_mla_v4_decode_fused", + "flydsl_mla_v4_decode_supported", + "flydsl_mla_v4_grouped_pays", + "flydsl_mla_v4_grouped_verify_meta", +] diff --git a/python/sglang/kernels/ops/attention/dsv4/flydsl_v4nm/kernels/__init__.py b/python/sglang/kernels/ops/attention/dsv4/flydsl_v4nm/kernels/__init__.py new file mode 100644 index 0000000000..b838e94c7e --- /dev/null +++ b/python/sglang/kernels/ops/attention/dsv4/flydsl_v4nm/kernels/__init__.py @@ -0,0 +1,2 @@ +# TODO(aiter#5882): remove once ROCm/aiter#5882 is in the aiter pin +"""FlyDSL kernel bodies of the in-tree ROCm/aiter#5882 copy (see the package).""" diff --git a/python/sglang/kernels/ops/attention/dsv4/flydsl_v4nm/kernels/mla_v4_decode_common.py b/python/sglang/kernels/ops/attention/dsv4/flydsl_v4nm/kernels/mla_v4_decode_common.py new file mode 100644 index 0000000000..e1f847e97c --- /dev/null +++ b/python/sglang/kernels/ops/attention/dsv4/flydsl_v4nm/kernels/mla_v4_decode_common.py @@ -0,0 +1,252 @@ +# SPDX-License-Identifier: MIT +# Copyright (C) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. +# +# Vendored from ROCm/aiter#5882 (https://github.com/ROCm/aiter/pull/5882), +# commit 03359c43df (03359c43df40abf91ac38a2040c9b8e284d92297), +# file aiter/ops/flydsl/kernels/mla_v4_decode_common.py. +# Unchanged except for import paths and sglang formatting (isort, ruff format). +# TODO(aiter#5882): remove once ROCm/aiter#5882 is in the aiter pin + +"""Shared helpers of the FlyDSL v4 nm MLA decode (mla_v4_decode_gfx950.py): math +intrinsics, cross-lane butterflies, the global atomics / loads / fences of the +cross-split merge protocol, and the mxfp8 (ceil-e8m0) block scale.""" + +import flydsl.compiler as flyc +import flydsl.expr as fx +from aiter.ops.flydsl.kernels import communication_ops_utils as cu +from flydsl._mlir.dialects import llvm as _llvm_d +from flydsl.expr import arith as _arith +from flydsl.expr.typing import T + +FP8_MAX = 448.0 +EX_FLOOR = 94 # smallest e8m0 block scale emitted: 2^(94 - 127) + + +def exp2(x): + return fx.Float32(fx.rocdl.exp2(T.f32, fx.Float32(x).ir_value())) + + +def rcp(x): + return fx.Float32(fx.rocdl.rcp(T.f32, fx.Float32(x).ir_value())) + + +def ldexp(x, e): + return fx.Float32( + _llvm_d.call_intrinsic( + T.f32, + "llvm.ldexp.f32.i32", + [fx.Float32(x).ir_value(), fx.Int32(e).ir_value()], + [], + [], + ) + ) + + +def fabs_f32(x): + return fx.Float32( + _llvm_d.call_intrinsic( + T.f32, "llvm.fabs.f32", [fx.Float32(x).ir_value()], [], [] + ) + ) + + +def maxnum_f32(a, b): + return fx.Float32( + _llvm_d.call_intrinsic( + T.f32, + "llvm.maxnum.f32", + [fx.Float32(a).ir_value(), fx.Float32(b).ir_value()], + [], + [], + ) + ) + + +def fabs_bits(x): + return fabs_f32(x).bitcast(fx.Int32) + + +def udiv(a, b): + return fx.Int32(_arith.divui(fx.Int32(a).ir_value(), fx.Int32(b).ir_value())) + + +def urem(a, b): + return fx.Int32(_arith.remui(fx.Int32(a).ir_value(), fx.Int32(b).ir_value())) + + +def smax_i32(a, b): + return (a > b).select(a, b) + + +def smin_i32(a, b): + return (a < b).select(a, b) + + +def dpp_i32(x, ctrl): + return fx.Int32( + fx.rocdl.update_dpp( + T.i32, fx.Int32(0).ir_value(), fx.Int32(x).ir_value(), ctrl, 0xF, 0xF, True + ) + ) + + +def plswap(x, width, op2): + st = _llvm_d.StructType.get_literal([T.i32, T.i32]) + op = fx.rocdl.permlane16_swap if width == 16 else fx.rocdl.permlane32_swap + r = op(st, fx.Int32(x).ir_value(), fx.Int32(x).ir_value(), False, False) + a = fx.Int32(_llvm_d.extractvalue(T.i32, r, [0])) + b = fx.Int32(_llvm_d.extractvalue(T.i32, r, [1])) + return op2(a, b) + + +_DPP_XOR = {1: 0xB1, 2: 0x4E, 4: 0x141, 8: 0x128} + + +def butterfly_i32(x, nlanes, op2, first=1): + k = first + while k < nlanes * first: + if k <= 8: + x = op2(x, dpp_i32(x, _DPP_XOR[k])) + else: + x = plswap(x, k, op2) + k *= 2 + return x + + +def mxfp8_exp(amb): + # e8m0 of ceil(amax / 448) from amax's bits: 448 = 1.75 * 2^8, so a mantissa + # above 0.75 (0x600000) rounds the exponent up + ea = amb >> fx.Int32(23) + ex = ( + ea + - fx.Int32(8) + + ((amb & fx.Int32(0x7FFFFF)) > fx.Int32(0x600000)).select( + fx.Int32(1), fx.Int32(0) + ) + ) + ex = (ex < fx.Int32(EX_FLOOR)).select(fx.Int32(EX_FLOOR), ex) + return (ea >= fx.Int32(255)).select(fx.Int32(254), ex) + + +def fp8_scaled(y, nexp): + v = ldexp(y, nexp).maximumf(fx.Float32(-FP8_MAX)) + return (v > fx.Float32(FP8_MAX)).select(fx.Float32(FP8_MAX), v) + + +def xcc_id(): + return fx.Int32( + _llvm_d.InlineAsmOp( + T.i32, + [], + "s_getreg_b32 $0, hwreg(HW_REG_XCC_ID)", + "=s", + has_side_effects=True, + ).res + ) + + +def pin_v64(x): + return fx.Int64( + _llvm_d.InlineAsmOp( + T.i64, [fx.Int64(x).ir_value()], "; pin $0", "=v,0", has_side_effects=True + ).res + ) + + +def gload(addr_i64, n): + return fx.generic_load(cu._to_ptr_global(addr_i64, fx.Float32, 4 * n), count=n) + + +def inv_l1(): + _llvm_d.InlineAsmOp(None, [], "buffer_inv sc0", "", has_side_effects=True) + + +def rmw_add64_agent(addr_i64, val): + return fx.atomic_add( + cu._to_ptr_global(addr_i64, fx.Int64, 8), fx.Int64(val), syncscope="agent" + ) + + +def ld64_agent(addr_i64, volatile=True): + return fx.generic_load( + cu._to_ptr_global(addr_i64, fx.Int64, 8), + volatile=volatile, + memory_order=fx.AtomicOrdering.Monotonic, + syncscope=fx.rocdl.SyncScope.AgentOneAs, + ) + + +def st_raw(addr_i64, value, align): + fx.generic_store(cu._to_ptr_global(addr_i64, alignment=align), value) + + +def st_agent(addr_i64, value, align): + fx.generic_store( + cu._to_ptr_global(addr_i64, alignment=align), + value, + memory_order=fx.AtomicOrdering.Monotonic, + syncscope=fx.rocdl.SyncScope.AgentOneAs, + ) + + +def st_agent_vec(addr_i64, value, nbytes): + # sc1 store: written through to the device coherence point (no L2 write-back + # needed); s_nop 1 covers the store-data VGPR hazard the backend does not see + if nbytes == 4: + v = fx.Float32(value).bitcast(fx.Int32) + op = "global_store_dword $0, $1, off sc1\n\ts_nop 1" + else: + assert nbytes == 16, nbytes + v = fx.Vector(value).bitcast(fx.Int32) + op = "global_store_dwordx4 $0, $1, off sc1\n\ts_nop 1" + _llvm_d.InlineAsmOp( + None, + [fx.Int64(addr_i64).ir_value(), _arith.unwrap(v)], + op, + "v,v,~{memory}", + has_side_effects=True, + ) + + +def ld_agent_f32(addr_i64, n): + if n == 1: + return ld_agent_i32(addr_i64, volatile=False).bitcast(fx.Float32) + parts = [ + ld64_agent(addr_i64 + fx.Int64(8 * i), volatile=False) for i in range(n // 2) + ] + return fx.Vector.from_elements(parts, fx.Int64).bitcast(fx.Float32) + + +@flyc.jit +def spin_all_arrived(addr_i64, full, bit): + cur = ld64_agent(addr_i64) + while ((cur & full) != full) & ((cur & bit) != fx.Int64(0)): + cur = ld64_agent(addr_i64) + return cur + + +def ld_agent_i32(addr_i64, volatile=True): + return fx.generic_load( + cu._to_ptr_global(addr_i64), + volatile=volatile, + memory_order=fx.AtomicOrdering.Monotonic, + syncscope=fx.rocdl.SyncScope.AgentOneAs, + ) + + +def tickets_state(v, want): + z = fx.Int64(fx.rocdl.ballot(T.i64, (v == fx.Int32(0)).ir_value())) + o = fx.Int64(fx.rocdl.ballot(T.i64, (v != want).ir_value())) + return (z == fx.Int64(0)).select(fx.Int32(1), fx.Int32(0)) | ( + (o == fx.Int64(0)).select(fx.Int32(2), fx.Int32(0)) + ) + + +@flyc.jit +def spin_tickets_state(addr_lane_i64, v, max_polls, want): + r = tickets_state(v, want) + k = fx.Int32(0) + while ((r & fx.Int32(1)) == fx.Int32(0)) & (k < fx.Int32(max_polls)): + r = tickets_state(ld_agent_i32(addr_lane_i64), want) + k = k + fx.Int32(1) + return r diff --git a/python/sglang/kernels/ops/attention/dsv4/flydsl_v4nm/kernels/mla_v4_decode_gfx950.py b/python/sglang/kernels/ops/attention/dsv4/flydsl_v4nm/kernels/mla_v4_decode_gfx950.py new file mode 100644 index 0000000000..87acfe5edd --- /dev/null +++ b/python/sglang/kernels/ops/attention/dsv4/flydsl_v4nm/kernels/mla_v4_decode_gfx950.py @@ -0,0 +1,5271 @@ +# SPDX-License-Identifier: MIT +# Copyright (C) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. +# +# Vendored from ROCm/aiter#5882 (https://github.com/ROCm/aiter/pull/5882), +# commit 03359c43df (03359c43df40abf91ac38a2040c9b8e284d92297), +# file aiter/ops/flydsl/kernels/mla_v4_decode_gfx950.py. +# Unchanged except for import paths and sglang formatting (isort, ruff format). +# TODO(aiter#5882): remove once ROCm/aiter#5882 is in the aiter pin + +"""DeepSeek-V4 fp8 MLA decode (v4 nm contract) in one FlyDSL launch on gfx950. + +Replaces the asm mla_a8w8_qh64_qseqlen1_gqaratio64_nm kernel + its split merge. +q / kv rows: 448 e4m3 + 7 e8m0 scales (each stored twice at bytes 448..461; bytes +462..511 never read), bf16 rope, page size 1, one kv head. Optional per-q-row key +ranges (q_kv_bounds, grouped DSpark verify) and a DeepSeek-V4 epilogue (inverse RoPE + +wo_a mxfp8 quant). + +Grid: ceil(B / 8) * 8 * NRG * S CTAs; bid % 8 is the XCD, so all CTAs of a sequence +share one L2 (SPREAD / XR / SK / ADAPT change the map). Split s walks key tiles +[s * tps, (s + 1) * tps); kv lengths are never compile-time values. + +Tile loops (V4Cfg.LAYOUT; rows = 16 * RB q rows of one sequence per CTA): + K 4 waves, 64-key tiles: fp8 QK with the row e8m0 scales as MFMA block scales, V + dequantized once to bf16 in LDS, lazy online softmax, each wave owns 128 of the + 512 output columns. + RQ 32-row waves, 32-key tiles (64 or 128-row CTAs): 32x32 fp8 QK, PV over all 512 + columns from AGPR accumulators, K / V tiles DMA'd global -> LDS ahead. + RD the RQ QK with PV split over dv; Q16: two wave roles, 16-row QK waves (QK + + softmax of step s + 1) next to 4 PV waves (PV of step s + dequant), one barrier + per tile. +A variant's SUB bodies are other K-layout (or, under Q16, 32-row Q16) variants in the +same kernel; each sequence runs the first whose key bound covers its stream. + +Cross-split merge (S > 1), co-residency free and graph replayable: per merge group one +SLOT_BYTES slot = arrive | stay mask (i64) + one start ticket per split. Every CTA +stores its ticket at entry; at the arrive wave 0 reads the tickets (bounded re-polls): +all started -> stay and merge its own slice, else leave the slice to the last +arriver. Release: vmcnt(0) if all siblings are on this XCD (or partials were written +through), else an agent-scope L2 write-back; then one returning i64 atomic add. The +last arriver zeroes the slot and merges the slices of every leaver. No CTA ever waits +for one that has not started. +""" + +import collections +import functools +import hashlib +import os +import types +from dataclasses import dataclass, replace + +import flydsl.compiler as flyc +import flydsl.expr as fx +from aiter.ops.flydsl.kernels import buffer_ops +from aiter.ops.flydsl.kernels import communication_ops_utils as cu +from aiter.ops.flydsl.kernels.kernels_common import LOG2E +from flydsl._mlir import ir as _ir +from flydsl._mlir.dialects import llvm as _llvm_d +from flydsl._mlir.dialects import rocdl as _rocdl_d +from flydsl.expr import arith as _arith +from flydsl.expr import math as fly_math +from flydsl.expr.typing import T + +from .mla_v4_decode_common import ( + butterfly_i32, + exp2, + fabs_bits, + fabs_f32, + fp8_scaled, + gload, + inv_l1, + ld64_agent, + ld_agent_f32, + ld_agent_i32, + maxnum_f32, + mxfp8_exp, + pin_v64, + plswap, + rcp, + rmw_add64_agent, + smax_i32, + smin_i32, + spin_all_arrived, + spin_tickets_state, + st_agent, + st_agent_vec, + st_raw, + udiv, + urem, + xcc_id, +) + +WAVE = 64 +NWAVES = 4 +NTHREADS = WAVE * NWAVES +NOPE = 448 +ROPE = 64 +DV = 512 +ROWB = 512 +KT_K = 64 # keys per K-layout tile (V4Cfg.KT) +VPITCH = DV + 8 +PPITCH = KT_K + 8 +MAX_S = 32 +SCALE_LOG2E = DV**-0.5 * LOG2E +# merge-group slot: arrive | stay mask (i64 @ A_OFF), start tickets (i32 per split @ +# TK_OFF, XCC_ID + 1); WS_HDR bytes of slots, zero between launches +SLOT_BYTES = 256 +WS_HDR = 1 << 20 +MAX_GROUPS = WS_HDR // SLOT_BYTES +A_OFF, TK_OFF = 0, 64 +# bounded re-polls of the start tickets before a CTA leaves its slice +START_POLLS = 8 +# non-temporal load cache policy +CPOL_NT = 2 +# RQ / RD LDS ring slot bytes: fp8 keys, rope keys, V (one 32-key tile each) +RQ_KF = 32 * 32 * 16 +RQ_KR = 8 * 32 * 16 +RQ_V = 56 * 32 * 16 + + +def rq_rings(rq, rows): + """(key tiles in flight, fp8-key / rope-key / V / key-index ring slots).""" + if rq and rows == 64: + return 2, 2, 3, 1, 3 + return 3, 4, 4, 2, 4 + + +def rq_lds_layout(rd, rows, nkd, rings, npb, nml): + _, nkf, nkr, nv, nix = rings + kr_off = nkf * RQ_KF + v_off = kr_off + nkr * RQ_KR + ix_off = v_off + nv * RQ_V + p_off = ix_off + nix * (2 * 256 * 3 if nkd == 3 else 4 * 256 * 2) + a_off = p_off + npb * rows * 80 + ml_off = a_off + npb * rows * 4 + total = (ml_off + nml * rows * 4) if rd else p_off + return kr_off, v_off, ix_off, p_off, a_off, ml_off, total + + +@dataclass(frozen=True) +class V4Cfg: + """One compiled variant (also the compile-cache key); plans in mla_v4_decode.py. + + H, MSQ: q heads, q rows per sequence (grouped verify: per group). RB: 16-row blocks + per CTA (NRG = H * MSQ / rows row groups). S: kv splits per (sequence, row group). + EPI: "bf16" output or "invrope_mxfp8" (inverse RoPE + wo_a mxfp8 quant). + KC: coalesced K-layout KV gather. LAYOUT: tile loop (module docstring). + MASK: "bounds" reads q_kv_bounds. ADAPT (K): run-time split count per sequence from + its tiles and the launch's, for ADAPT CTA slots. MINT: minimum tiles per split. + Q16 (RD): the two-role loop. SHORT (Q16, SK): streams of <= SHORT tiles run unsplit; + SHORTM: also capped by the launch's mean stream; SHORTF: <= SHORTF tiles always. + PF16 (Q16): fp16 split partials, power-of-2 scaled. XR: a sequence's row groups on + consecutive XCDs. SUB: ((max keys, body), ...) stream-length bodies. + SPREAD: a sequence's splits over all XCDs (small batches, written-through + partials). PFN (Q16): after its last tile a CTA prefetches the metadata, q rows and + first key indices of CTA bid + PFN into L2. OWN (RQ): split j merges column + quarters j (mod S), its own partial stays in registers. SK (K): grid of SK CTAs, + SKD splits per sequence, or, when the longest split would set the tail (runs of + >= SKT tiles, or one of > SKT tiles, 2 runs and 4 shares), the batch's keys cut into + SK equal runs; SKMEAN: only streams longer than the mean split. PV16R = (lo, hi) + (unsplit Q16): streams of lo < keys <= hi use 16x16x32 PV MFMAs (less energy per + key under the power cap). TAIL: units (sequences, XR: row groups) past the last + complete XCD octet spread over all XCDs, their partials written through. + IDLE = (x, x2, f, w) (Q16): one CTA per (sequence, row group) on an unsplit body, + plus CTAs up to x + x2 per XCD taking splits 1 .. S - 1 of the XCD's longest + streams (ranked on device) of > f tiles and longer than the mean; past x only of + > w tiles. + """ + + H: int = 16 + MSQ: int = 1 + RB: int = 1 + S: int = 1 + EPI: str = "bf16" + KC: bool = False + LAYOUT: str = "K" + MASK: str = "none" + ADAPT: int = 0 + MINT: int = 1 + Q16: bool = False + SHORT: int = 0 + SHORTM: bool = False + SHORTF: int = 0 + PF16: bool = False + XR: bool = False + SUB: tuple = () + SPREAD: bool = False + PFN: int = 0 + OWN: bool = False + SK: int = 0 + SKD: int = 1 + SKMEAN: bool = False + SKT: int = 12 + PV16R: tuple = () + TAIL: bool = False + IDLE: tuple = () + + def __post_init__(self): + assert self.H * self.MSQ % (16 * self.RB) == 0 and 1 <= self.S <= MAX_S, self + assert not (self.TAIL and (self.SPREAD or self.ADAPT)), self + assert not (self.TAIL and (self.SHORTM or self.LAYOUT == "RQ")), self + + @property + def KT(self): + return 64 if self.LAYOUT == "K" else 32 + + def ws_seq(self): + if self.S == 1: + return 0, 0 + sp = (self.S + 3) // 4 * 4 + return self.NRG, self.NRG * self.rows * (self.S * DV * 4 + sp * 4) + + def ws_need(self, B): + g = b = 0 + for c in (self,) + tuple(sub for _, sub in self.SUB): + g1, b1 = c.ws_seq() + g, b = g + g1, b + b1 + bpad = -(-B // 8) * 8 + return bpad * g, bpad * b + + @property + def cps(self): + return max([self.NRG * self.S] + [sub.NRG * sub.S for _, sub in self.SUB]) + + @property + def NRG(self): + return self.H * self.MSQ // (16 * self.RB) + + @property + def rows(self): + return 16 * self.RB + + @property + def nthreads(self): + if self.LAYOUT == "RQ": + return WAVE * self.RB // 2 + if self.LAYOUT == "RD" and self.Q16: + return WAVE * (4 + self.RB) + if self.LAYOUT in ("K", "RD"): + return NTHREADS + return WAVE * self.RB + + +def plswap_f(x, width, op2): + def op2f(a, b): + return op2(a.bitcast(fx.Float32), b.bitcast(fx.Float32)) + + return plswap(fx.Float32(x).bitcast(fx.Int32), width, op2f) + + +def _fmax(a, b): + return a.maximumf(b) + + +def _fadd(a, b): + return a + b + + +def _div_const(a, d): + if d & (d - 1) == 0: + return a >> fx.Int32(d.bit_length() - 1) + return udiv(a, fx.Int32(d)) + + +def _ult(a, b): + return fx.Boolean(_arith.cmpi(_arith.CmpIPredicate.ult, a, b)) + + +def _rem_const(a, d): + if d & (d - 1) == 0: + return a & fx.Int32(d - 1) + return urem(a, fx.Int32(d)) + + +def xcd_ranks(indptr_ptr, nseq, x, lane, nmax): + """Lane l: stream j = 8 l + x, whether it exists, and its rank among those streams + (longest first, ties by j); at most nmax of them.""" + j = (lane << fx.Int32(3)) | x + ok = j < nseq + jc = ok.select(j, fx.Int32(0)) + a0 = fx.Int32(fx.ptr_load(indptr_ptr + fx.Int64(jc))) + n = fx.Int32(fx.ptr_load(indptr_ptr + fx.Int64(jc) + fx.Int64(1))) - a0 + n = ok.select(n, fx.Int32(-1)) + rk = fx.Int32(0) + for m in range(nmax): + nm = fx.Int32(fx.rocdl.readlane(T.i32, n.ir_value(), fx.Int32(m).ir_value())) + first = (nm > n) | ((nm == n) & (lane > fx.Int32(m))) + rk = rk + first.select(fx.Int32(1), fx.Int32(0)) + return j, ok, rk + + +def _sload(ptr, i): + return fx.Int32( + buffer_ops.buffer_load( + buffer_ops.create_buffer_resource_from_addr( + fx.Int64(fx.ptrtoint(ptr)).ir_value() + ), + i, + vec_width=1, + is_scalar=True, + ) + ) + + +# one opaque SGPR use of all xs: loads issued before a branch stay one round trip +def _pin_s(xs): + vs = [x.ir_value() for x in xs] + n = len(vs) + r = _llvm_d.InlineAsmOp( + _llvm_d.StructType.get_literal([v.type for v in vs]), + vs, + "; " + " ".join(f"${i}" for i in range(n)), + ",".join(["=s"] * n + [str(i) for i in range(n)]), + has_side_effects=True, + ).res + return [ + type(x)(_llvm_d.extractvalue(v.type, r, [i])) + for i, (x, v) in enumerate(zip(xs, vs)) + ] + + +def _tail_map(bid, nseq, units, per): + # V4Cfg.TAIL: -> (unit, CTA of the unit); complete octets of units keep unit % 8 as + # the XCD, the rest take consecutive CTAs (all XCDs) + full = (nseq * fx.Int32(units)) & fx.Int32(-8) + ncol = full * fx.Int32(per) + bj = bid >> fx.Int32(3) + k = _div_const(bj, per) + t = bid - ncol + kt = _div_const(t, per) + colo = bid < ncol + return ( + colo.select((k << fx.Int32(3)) | (bid & fx.Int32(7)), full + kt), + colo.select(bj - k * fx.Int32(per), t - kt * fx.Int32(per)), + ) + + +# one merge round's rows of a split slice (see slice_addrs) +_SliceRow = collections.namedtuple( + "_SliceRow", "act head qpos d0 la pa sa frr cblk psa" +) + + +class _AccRow: + def __init__(self, acc, npb, rb, lazy): + self._acc, self._npb, self._rb = acc, npb, rb + self._runs = None if lazy else [self._run(n) for n in range(16)] + + def _run(self, n): + blk = self._acc[(n // 4) * self._npb + self._rb] + return fx.Vector.from_elements( + [blk[4 * (n % 4) + i] for i in range(4)], fx.Float32 + ) + + def __getitem__(self, n): + return self._run(n) if self._runs is None else self._runs[n] + + +def _emitter(cfg: V4Cfg, ws_prev=(0, 0)): + # emit(): split_range / load_q / protocol_entry, the LAYOUT tile loop (RQ, then + # RD / Q16, K), then direct_epi (unsplit) or merged_epi (cross-split merge) + H, MSQ, RB, S = cfg.H, cfg.MSQ, cfg.RB, cfg.S + HB = H // 16 + NRG = cfg.NRG + ROWS = cfg.rows + EPI_Q = cfg.EPI == "invrope_mxfp8" + KC = cfg.KC + RQ = cfg.LAYOUT == "RQ" + RD = cfg.LAYOUT == "RD" + KL = cfg.LAYOUT == "K" + OWN = cfg.OWN and S > 1 + RQM = RQ and (RB == 4 or OWN) + PF16 = cfg.Q16 and cfg.PF16 + PEB = 2 if PF16 else 4 + BOUNDS = cfg.MASK == "bounds" + NT = cfg.nthreads + KTL = cfg.KT + KT_SH = KTL.bit_length() - 1 + NW = NT // WAVE + RINGS = rq_rings(RQ, cfg.rows) + DD, NKF, NKR, NVB_RQ, NIX = RINGS + NROLE = 4 // NW if RQ else 1 + Q16 = cfg.Q16 + # key rows nobody else reads (one row group) stream in as non-temporal loads: they + # evict no dirty last-level-cache lines that would be written back mid-kernel + KV_NT = Q16 and NRG == 1 + PV16R = cfg.PV16R if (Q16 and S == 1) else () + RD_NQW = cfg.RB if Q16 else cfg.rows // 32 + RD_PVD = 2 + ( + RQ_KR_OFF, + RQ_V_OFF, + RQ_IX_OFF, + RD_P_OFF, + RD_A_OFF, + RD_ML_OFF, + RQD_LDS, + ) = rq_lds_layout( + RD, + cfg.rows, + 3 if (RD and RD_NQW < 4 and not Q16) else 2, + RINGS, + 2 if Q16 else 1, + 3 if Q16 else 2, + ) + MINT = cfg.MINT + ADAPT = cfg.ADAPT if S > 1 else 0 + SHORT = cfg.SHORT if S > 1 and not cfg.SK else 0 + SHORTM = cfg.SHORTM and SHORT > 0 + SHORTF = cfg.SHORTF if SHORTM else 0 + SK = cfg.SK if S > 1 else 0 + DYN = ADAPT or SK + QROW_EPI = MSQ > 1 + MERGED = S > 1 + Q16_OSC = Q16 and (not MERGED or SHORT > 0) + WTP = (cfg.SPREAD or cfg.TAIL and not cfg.SK) and MERGED + SP = (S + 3) // 4 * 4 + NQB = ROWS * 4 + lse_chunks = [] + k = 0 + while k < S: + n = 4 if S - k >= 4 else (2 if S - k >= 2 else 1) + lse_chunks.append((k, n)) + k += n + bpc = (NQB + S - 1) // S + if DYN: + bpc = (NQB + 1) // 2 + EPT = 2 if bpc <= 4 else 4 + TPB = 128 // EPT + BPR = NT // TPB + NRND = (bpc + BPR - 1) // BPR + MCH = max(1, min(NRND, 64 // (S * EPT))) + SKP0 = min(SP, -(-max(4, cfg.SKD) // 4) * 4) + if SK: + MCH = max(1, min(NRND, 64 // (SKP0 * EPT), -(-NQB // (cfg.SKD * BPR)))) + MCHUNKS = list(range(0, NRND, MCH)) + FULL = (1 << S) - 1 + FULL32 = FULL - (1 << 32) if FULL >= 1 << 31 else FULL + + if RQ or RD: + + @fx.struct + class Smem: + buf: fx.Array[fx.Uint8, RQD_LDS, 16] + flag: fx.Array[fx.Int32, 4, 16] + + else: + + @fx.struct + class Smem: + vlds: fx.Array[fx.BFloat16, KT_K * VPITCH, 16] + plds: fx.Array[fx.BFloat16, ROWS * PPITCH, 16] + rmax: fx.Array[fx.Float32, NWAVES * ROWS, 16] + rsum: fx.Array[fx.Float32, NWAVES * ROWS, 16] + flag: fx.Array[fx.Int32, 4, 16] + + name = ( + f"mla_v4_decode_h{H}_q{MSQ}_rb{RB}_s{S}_{cfg.EPI}" + + ("_kc" if KC else "") + + ("_rq" if RQ else "") + + ("_rd" if RD else "") + + ("q16" if Q16 else "") + + ("_xr" if cfg.XR else "") + + (f"_sh{cfg.SHORT}" if cfg.SHORT and S > 1 else "") + + ("m" if cfg.SHORTM and S > 1 else "") + + (f"f{cfg.SHORTF}" if cfg.SHORTM and cfg.SHORTF and S > 1 else "") + + ("_pf16" if cfg.PF16 and S > 1 else "") + + ("_bnd" if BOUNDS else "") + + (f"_ad{cfg.ADAPT}m{cfg.MINT}" if ADAPT else "") + + (f"_sk{SK}d{cfg.SKD}t{cfg.SKT}" if SK else "") + + ("m" if SK and cfg.SKMEAN else "") + + ("_own" if OWN else "") + + (f"_pv{PV16R[0]}_{PV16R[1]}" if PV16R else "") + + ("_tl" if cfg.TAIL else "") + ) + + def map_bid(bid, nseq): + xcd = bid & fx.Int32(7) + bj = bid >> fx.Int32(3) + if fx.const_expr(cfg.TAIL and cfg.XR): + unit, split = _tail_map(bid, nseq, NRG, S) + return _div_const(unit, NRG), _rem_const(unit, NRG), split + if fx.const_expr(cfg.TAIL): + seq, c = _tail_map(bid, nseq, 1, NRG * S) + rg = _div_const(c, S) + return seq, rg, c - rg * fx.Int32(S) + if fx.const_expr(ADAPT or (SHORT and not cfg.XR)): + # split-major: the CTAs of short (unsplit) streams dispatch first + gpx = ((nseq + fx.Int32(7)) >> fx.Int32(3)) * fx.Int32(NRG) + split = udiv(bj, gpx) + k2 = bj - split * gpx + elif fx.const_expr(cfg.XR): + split = _rem_const(bj, S) + unit = (_div_const(bj, S) << fx.Int32(3)) | xcd + return _div_const(unit, NRG), _rem_const(unit, NRG), split + else: + split = _rem_const(bj, S) + k2 = _div_const(bj, S) + rg = _rem_const(k2, NRG) + seq = (_div_const(k2, NRG) << fx.Int32(3)) | xcd + return seq, rg, split + + @flyc.jit + def emit( + nseq: fx.Int32, + num_rows: fx.Int32, + indptr_ptr: fx.Pointer, + idx_ptr: fx.Pointer, + q_ptr: fx.Pointer, + kv_ptr: fx.Pointer, + kvr_ptr: fx.Pointer, + qr_ptr: fx.Pointer, + qo_ptr: fx.Pointer, + pos_ptr: fx.Pointer, + ws_ptr: fx.Pointer, + sink_ptr: fx.Pointer, + out_ptr: fx.Pointer, + xs_ptr: fx.Pointer, + fr_ptr: fx.Pointer, + out_s0: fx.Int32, + out_s1: fx.Int32, + hdr_ptr: fx.Pointer, + bnd_ptr: fx.Pointer, + lds, + seq, + rg, + split, + kvm=None, + skr=None, + ): + v16u8 = fx.Vector.make_type(16, fx.Uint8) + v4f32 = fx.Vector.make_type(4, fx.Float32) + v8bf16 = fx.Vector.make_type(8, fx.BFloat16) + v4bf16 = fx.Vector.make_type(4, fx.BFloat16) + v2bf16 = fx.Vector.make_type(2, fx.BFloat16) + zero_i = fx.Int32(0) + true_ = zero_i == fx.Int32(0) + false_ = zero_i != fx.Int32(0) + ninf = fx.Float32(float("-inf")) + qnan = fx.Float32(float("nan")) + scale_log2e = fx.Float32(SCALE_LOG2E) + + tid = fx.Int32(fx.thread_idx.x) + wave = tid // fx.Int32(WAVE) + lane = tid % fx.Int32(WAVE) + m = lane % fx.Int32(16) + g = lane // fx.Int32(16) + group = seq * fx.Int32(NRG) + rg + ws_base = fx.Int64(fx.ptrtoint(ws_ptr)) + bpad = fx.Int64(((nseq + fx.Int32(7)) >> fx.Int32(3)) * fx.Int32(8)) + ngroups = bpad * fx.Int64(NRG) + hdr_a = fx.Int64(fx.ptrtoint(hdr_ptr)) + part_base = ws_base + bpad * fx.Int64(ws_prev[1]) + lse_base = part_base + ngroups * fx.Int64(S * ROWS * DV * PEB) + psc_base = lse_base + ngroups * fx.Int64(ROWS * SP * 4) + + if fx.const_expr(Q16): + # two waves per SIMD: pin the AGPR budget to the PV accumulators + _llvm_d.InlineAsmOp( + None, [], "; q16 agpr budget", "~{a127}", has_side_effects=True + ) + if fx.const_expr(RQ or RD): + p_rq = lds.buf.ptr + else: + p_v = lds.vlds.ptr + p_p = lds.plds.ptr + p_rmax = lds.rmax.ptr + p_rsum = lds.rsum.ptr + p_flag = lds.flag.ptr + + def lds_st(ptr, idx, val): + fx.ptr_store(val, ptr + idx) + + def lds_ldf(ptr, idx): + return fx.Float32(fx.ptr_load(ptr + idx)) + + seq_c = (seq < nseq).select(seq, zero_i) + kv_md = None + kv_all = None + if fx.const_expr(kvm is not None and len(kvm) == 4): + q0, qlen = kvm[2], kvm[3] + elif fx.const_expr(EPI_Q and MSQ == 1): + q0 = seq_c + qlen = fx.Int32(1) + elif fx.const_expr(SK): + q0, qlen = skr[5] + else: + md = [ + _sload(ptr, seq_c + fx.Int32(j)) + for ptr in (qo_ptr, indptr_ptr) + for j in range(2) + ] + if fx.const_expr(SHORTM): + # the launch's key count in the same round trip + md += [_sload(indptr_ptr, nseq), _sload(indptr_ptr, zero_i)] + if fx.const_expr(kvm is None): + if fx.const_expr(not EPI_Q): + # (with the merge slots' kernarg) + md = _pin_s(md + [hdr_a] * MERGED) + hdr_a = md[-1] if MERGED else hdr_a + kv_md = (md[2], md[3] - md[2]) + if fx.const_expr(SHORTM): + kv_all = md[4] - md[5] + q0 = md[0] + qlen = md[1] - q0 + slot_base = hdr_a + bpad * fx.Int64(ws_prev[0] * SLOT_BYTES) + jb0 = rg * fx.Int32(RB) + active = (seq < nseq) & (_div_const(jb0, HB) < qlen) + + def split_range(sq): + if fx.const_expr(kvm is not None): + kv0, kvlen = kvm[0], kvm[1] + elif fx.const_expr(kv_md is not None): + kv0, kvlen = kv_md + else: + kv0 = fx.Int32(fx.ptr_load(indptr_ptr + fx.Int64(sq))) + kvlen = ( + fx.Int32(fx.ptr_load(indptr_ptr + fx.Int64(sq) + fx.Int64(1))) - kv0 + ) + if fx.const_expr(SK): + t0, t1, ns, lastseg, wthru, _, tps, ntiles = skr + return kv0, kvlen, ntiles, tps, t0, t1, ns, true_, lastseg, wthru + ntiles = (kvlen + fx.Int32(KTL - 1)) >> fx.Int32(KT_SH) + tps = _div_const(ntiles + fx.Int32(S - 1), S) + if fx.const_expr(MINT > 1): + tps = (tps < fx.Int32(MINT)).select(fx.Int32(MINT), tps) + if fx.const_expr(ADAPT): + kvall = fx.Int32(fx.ptr_load(indptr_ptr + fx.Int64(nseq))) - fx.Int32( + fx.ptr_load(indptr_ptr) + ) + wt = (kvall + nseq * fx.Int32(KTL // 2)) >> fx.Int32(KT_SH) + nsl = (nseq > fx.Int32(ADAPT)).select(nseq, fx.Int32(ADAPT)) + tgt = udiv(wt + nsl - fx.Int32(1), nsl) + ns = udiv(ntiles + tgt - fx.Int32(1), tgt) + if fx.const_expr(MINT > 1): + nsf = _div_const(ntiles, MINT) + ns = (ns > nsf).select(nsf, ns) + ns = (ns > fx.Int32(S)).select(fx.Int32(S), ns) + ns = (ns < fx.Int32(1)).select(fx.Int32(1), ns) + tps = udiv(ntiles + ns - fx.Int32(1), ns) + tps1 = (tps < fx.Int32(1)).select(fx.Int32(1), tps) + ns = (tps > zero_i).select( + udiv(ntiles + tps1 - fx.Int32(1), tps1), fx.Int32(1) + ) + live = split < ns + t0 = live.select(split * tps, zero_i) + t1r = t0 + tps + t1 = live.select((t1r < ntiles).select(t1r, ntiles), zero_i) + elif fx.const_expr(SHORT): + short = ntiles <= fx.Int32(SHORT) + if fx.const_expr(SHORTM): + kvall = kv_all + if fx.const_expr(kvall is None): + kvall = _sload(indptr_ptr, nseq) - _sload(indptr_ptr, zero_i) + kts = nseq * fx.Int32(KTL) + upto = ntiles <= udiv(kvall + kts - fx.Int32(1), kts) + if fx.const_expr(SHORTF): + upto = upto | (ntiles <= fx.Int32(SHORTF)) + short = short & upto + ns = short.select(fx.Int32(1), fx.Int32(S)) + live = (short == false_) | (split == zero_i) + t0 = short.select(zero_i, split * tps) + t1r = t0 + tps + t1 = short.select(ntiles, (t1r < ntiles).select(t1r, ntiles)) + else: + ns, live = fx.Int32(S), true_ + t0 = split * tps + t1r = t0 + tps + t1 = (t1r < ntiles).select(t1r, ntiles) + return kv0, kvlen, ntiles, tps, t0, t1, ns, live, true_, false_ + + sr_pre = None + if fx.const_expr(ADAPT or SHORT): + sr_pre = split_range(seq_c) + active = active & sr_pre[7] + + if active: + + def load16(ptr, off): + return fx.ptr_load(ptr + fx.Int64(off), result_type=v16u8).bitcast( + fx.Int32 + ) + + def join8(lo, hi): + return fx.Vector.from_elements( + [lo[i] for i in fx.range_constexpr(4)] + + [hi[i] for i in fx.range_constexpr(4)], + fx.Int32, + ) + + zero4 = fx.Vector.filled(4, 0, fx.Int32) + + kv0, kvlen, ntiles, tps, t0, t1, ns, live, sk_last, sk_wt = ( + sr_pre if (ADAPT or SHORT) else split_range(seq) + ) + ibase = (kvlen > zero_i).select( + fx.Int64(fx.ptrtoint(idx_ptr)) + fx.Int64(kv0) * fx.Int64(4), + fx.Int64(fx.ptrtoint(indptr_ptr)) + fx.Int64(seq) * fx.Int64(4), + ) + + def ld_idx(off): + return fx.Int32( + fx.generic_load( + cu._to_ptr_global(ibase + fx.Int64(off) * fx.Int64(4)) + ) + ) + + rb_qrow, rb_h0, rb_ok = [], [], [] + NRBW = 1 if RQ else (ROWS // 32 if RD else RB) + if fx.const_expr(RQ or RD): + wv = fx.Int32(fx.rocdl.readfirstlane(T.i32, wave.ir_value())) + for rb in fx.range_constexpr(NRBW): + if fx.const_expr(RQ): + rq_row = (jb0 + wv * fx.Int32(2)) * fx.Int32(16) + ( + lane & fx.Int32(31) + ) + qpos = rq_row >> fx.Int32(H.bit_length() - 1) + h0 = (rq_row & fx.Int32(H - 1)) - (lane & fx.Int32(31)) + else: + if fx.const_expr(RD): + jb = jb0 + fx.Int32(2 * rb) + else: + jb = jb0 + fx.Int32(rb) + qpos = _div_const(jb, HB) + h0 = _rem_const(jb, HB) * fx.Int32(16) + rb_ok.append(qpos < qlen) + rb_qrow.append(q0 + (qpos < qlen).select(qpos, qlen - fx.Int32(1))) + rb_h0.append(h0) + + if fx.const_expr(BOUNDS and not Q16): + rb_bnd = [] + for rb in fx.range_constexpr(NRBW): + bv = fx.ptr_load( + bnd_ptr + fx.Int64(rb_qrow[rb]) * fx.Int64(4), + result_type=fx.Vector.make_type(4, fx.Int32), + ) + rb_bnd.append([fx.Int32(bv[i]) for i in range(4)]) + + def in_bounds(kk, bnd): + a0, a1, b0, b1 = bnd + return ((kk >= a0) & (kk < a1)) | ((kk >= b0) & (kk < b1)) + + if fx.const_expr(EPI_Q): + if fx.const_expr(QROW_EPI): + rb_pos = [ + fx.Int64(fx.ptr_load(pos_ptr + fx.Int64(rb_qrow[rb]))) + for rb in range(NRBW) + ] + else: + pos = fx.Int64(fx.ptr_load(pos_ptr + fx.Int64(seq))) + rb_pos = [pos] * NRBW + + gsh = g * fx.Int32(8) + + def _scl(word, cc): + b = (fx.Int32(word) >> gsh) & fx.Int32(0xFF) + if fx.const_expr(cc == 3): + b = (gsh >= fx.Int32(16)).select(fx.Int32(127), b) + return b + + def row_ok(raw): + inb = (raw >= zero_i) & (raw < num_rows) + return inb.select(raw, zero_i) + + def load_q(): + qfr, qscl, qrf = [], [], [] + for rb in fx.range_constexpr(NRBW): + qbase = ( + fx.Int64(rb_qrow[rb]) * fx.Int64(H) + fx.Int64(rb_h0[rb] + m) + ) * fx.Int64(ROWB) + fr_ = [] + for cc in fx.range_constexpr(4): + lo = load16( + q_ptr, + qbase + fx.Int64(cc * 128) + fx.Int64(g * fx.Int32(16)), + ) + if fx.const_expr(cc < 3): + hi = load16( + q_ptr, + qbase + + fx.Int64(cc * 128 + 64) + + fx.Int64(g * fx.Int32(16)), + ) + else: + hi = zero4 + fr_.append(join8(lo, hi)) + qfr.append(fr_) + qsc = load16(q_ptr, qbase + fx.Int64(NOPE)) + qscl.append([_scl(qsc[cc], cc) for cc in range(4)]) + qrbase = ( + fx.Int64(rb_qrow[rb]) * fx.Int64(H) + fx.Int64(rb_h0[rb] + m) + ) * fx.Int64(ROPE) + qrf.append( + [ + fx.ptr_load( + qr_ptr + + qrbase + + fx.Int64(g * fx.Int32(8) + fx.Int32(32 * j)), + result_type=v8bf16, + ) + for j in range(2) + ] + ) + return qfr, qscl, qrf + + # start ticket (fire and forget): siblings that see every ticket set may wait for + # this CTA, which is then known to be running + def protocol_entry(): + slot = slot_base + fx.Int64(group) * fx.Int64(SLOT_BYTES) + myx = xcc_id() & fx.Int32(7) + tk_on = tid == zero_i + if fx.const_expr(DYN or SHORT): + tk_on = tk_on & live & (ns > fx.Int32(1)) + if tk_on: + st_agent( + slot + fx.Int64(TK_OFF) + fx.Int64(split) * fx.Int64(4), + myx + fx.Int32(1), + 4, + ) + return slot, myx, ticket_lane(slot, lane) + + def ticket_lane(slot, lane): + nsp = ns if (DYN or SHORT) else fx.Int32(S) + return ( + slot + + fx.Int64(TK_OFF) + + fx.Int64((lane < nsp).select(lane, nsp - fx.Int32(1))) + * fx.Int64(4) + ) + + if fx.const_expr(RQ): + n32 = lane & fx.Int32(31) + h2 = lane >> fx.Int32(5) + r_cta = wv * fx.Int32(32) + n32 + row_st = r_cta < fx.Int32(ROWS) + + def stg(e): + return p_rq + e * fx.Int32(2) + + def row_meta(rc): + jbr = jb0 + (rc >> fx.Int32(4)) + qpr = _div_const(jbr, HB) + hdr = _rem_const(jbr, HB) * fx.Int32(16) + (rc & fx.Int32(15)) + okr = (qpr < qlen) & (rc < fx.Int32(ROWS)) + return q0 + (qpr < qlen).select(qpr, qlen - fx.Int32(1)), hdr, okr + + PE = 264 # staging row pitch (bf16): 256 + 8 against bank conflicts + + def emit_rows(val, wrow, d_lo, nblk, empty_r, pos_r): + ob = wv * fx.Int32(32 * PE) + NPB = min(nblk, 8) if not EPI_Q else nblk + RBY = NPB * 32 * (1 if EPI_Q else 2) + LPR = RBY // 16 + RPI = 64 // LPR + + def read_back(gaddr_of): + for rr in fx.range_constexpr(32 // RPI): + r_ = fx.Int32(rr * RPI) + lane // fx.Int32(LPR) + cb = (lane % fx.Int32(LPR)) * fx.Int32(16) + v_ = fx.ptr_load( + stg(ob + r_ * fx.Int32(PE) + (cb >> fx.Int32(1))), + result_type=v8bf16, + ) + qrr, hdr, okr = row_meta(wrow * fx.Int32(32) + r_) + if okr: + st_raw(gaddr_of(qrr, hdr) + fx.Int64(cb), v_, 16) + + def dvl(k, q_): + return fx.Int32(32 * k + 8 * q_) + h2 * fx.Int32(4) + + qrl, hdl, okl = row_meta(wrow * fx.Int32(32) + n32) + xrow = fx.Int64(qrl) * fx.Int64(H) + fx.Int64(hdl) + for bk in fx.range_constexpr(nblk // 4): + ys = [] + amb = zero_i + blk_ = d_lo // fx.Int32(4) + fx.Int32(bk) + is_rope = blk_ == fx.Int32(3) + for k in fx.range_constexpr(4 * bk, 4 * bk + 4): + for q_ in fx.range_constexpr(4): + x = val(k, q_).to(fx.BFloat16).to(fx.Float32) + if fx.const_expr(k % 4 >= 2): + dcol = ( + (d_lo + fx.Int32(k)) * fx.Int32(32) + + fx.Int32(8 * q_) + + h2 * fx.Int32(4) + ) + fcol = (dcol - fx.Int32(NOPE)) & fx.Int32(63) + fr = fx.ptr_load( + fr_ptr + + pos_r * fx.Int64(ROPE) + + fx.Int64(fcol), + result_type=v4f32, + ) + yy = [] + for pp in fx.range_constexpr(2): + c_ = fx.Float32(fr[2 * pp]) + sn = fx.Float32(fr[2 * pp + 1]) + xa = fx.Float32(x[2 * pp]) + xb = fx.Float32(x[2 * pp + 1]) + yy.append(is_rope.select(xa * c_ + xb * sn, xa)) + yy.append(is_rope.select(xb * c_ - xa * sn, xb)) + x = fx.Vector.from_elements(yy, fx.Float32) + y = x.to(fx.BFloat16).to(fx.Float32) + ys.append((k, q_, y)) + for i in fx.range_constexpr(4): + amb = smax_i32(amb, fabs_bits(fx.Float32(y[i]))) + amb = plswap(amb, 32, smax_i32) + ex = mxfp8_exp(amb) + ex = empty_r.select(fx.Int32(255), ex) + nexp = fx.Int32(127) - ex + for k, q_, y in ys: + qv = [fp8_scaled(fx.Float32(y[i]), nexp) for i in range(4)] + pkq = fx.rocdl.cvt_pk_fp8_f32( + T.i32, qv[0], qv[1], fx.Int32(0), False + ) + pkq = fx.rocdl.cvt_pk_fp8_f32( + T.i32, qv[2], qv[3], pkq, True + ) + pkq = empty_r.select(fx.Int32(0x7F7F7F7F), fx.Int32(pkq)) + fx.ptr_store( + fx.Vector.from_elements([pkq], fx.Int32).bitcast( + fx.BFloat16 + ), + stg( + ob + + n32 * fx.Int32(PE) + + (dvl(k, q_) >> fx.Int32(1)) + ), + ) + if okl & (h2 == zero_i): + fx.ptr_store( + fx.Uint8(ex), + xs_ptr + xrow * fx.Int64(4) + fx.Int64(blk_), + ) + xbase = fx.Int64(fx.ptrtoint(out_ptr)) + read_back( + lambda qr_, hd_: ( + xbase + + (fx.Int64(qr_) * fx.Int64(H) + fx.Int64(hd_)) + * fx.Int64(DV) + + fx.Int64(d_lo * fx.Int32(32)) + ) + ) + + def rq_part_val(acc_f, inv_lp, d, q_): + v = fx.Vector.from_elements( + [fx.Float32(acc_f[d][4 * q_ + i]) for i in range(4)], + fx.Float32, + ) + return v * inv_lp + + def rq_store_quarter(pw, acc_f, inv_lp, vq): + for d in fx.range_constexpr(4 * vq, 4 * vq + 4): + for q_ in fx.range_constexpr(4): + st_raw( + pw + fx.Int64((4 * d + q_) * 256 * PEB), + rq_part_val(acc_f, inv_lp, d, q_), + 4 * PEB, + ) + + def rq_part_addr(): + return part_base + ( + (fx.Int64(group) * fx.Int64(S) + fx.Int64(split)) + * fx.Int64(ROWS * DV) + + fx.Int64(wv * fx.Int32(64 * 256) + lane * fx.Int32(4)) + ) * fx.Int64(PEB) + + def rq_partials(m_f, l_f): + has = l_f > fx.Float32(0.0) + lf_ = l_f.maximumf(fx.Float32(1e-30)) + inv_lp = has.select(fx.Float32(1.0) / lf_, fx.Float32(0.0)) + lse_p = has.select(m_f + fly_math.log2(lf_), ninf) + return inv_lp, lse_p + + def rq_store(inv_lp, lse_p, acc_f, leave): + pw = rq_part_addr() + if t0 < t1: + okr = row_meta(r_cta)[2] + for vq in fx.range_constexpr(4): + other = fx.Int32(vq % S) != (split & fx.Int32(S - 1)) + if (leave | other) & okr: + rq_store_quarter(pw, acc_f, inv_lp, vq) + if row_st & (h2 == zero_i): + st_raw( + lse_base + + ( + (fx.Int64(group) * fx.Int64(ROWS) + fx.Int64(r_cta)) + * fx.Int64(SP) + + fx.Int64(split) + ) + * fx.Int64(4), + lse_p, + 4, + ) + + def merge_own(j, own): + rc = wv * fx.Int32(32) + n32 + qrl, hdl, okl = row_meta(rc) + ipm = fx.Float32( + _llvm_d.InlineAsmOp( + T.f32, + [fx.Float32(own_ip).ir_value()], + "; own $0", + "=v,0", + has_side_effects=True, + ).res + ) + if row_meta(wv * fx.Int32(32))[2]: + lane_ld = okl.select(lane, h2 * fx.Int32(32)) + rld = okl.select(rc, wv * fx.Int32(32)) + la = lse_base + ( + fx.Int64(group) * fx.Int64(ROWS) + fx.Int64(rld) + ) * fx.Int64(SP * 4) + lsv = [] + for k0, n in lse_chunks: + vv_ = fx.Vector(gload(la + fx.Int64(4 * k0), n)) + lsv += [fx.Float32(vv_[jx]) for jx in range(n)] + snk = fx.Float32( + fx.ptr_load(sink_ptr + fx.Int64(hdl)) + ) * fx.Float32(LOG2E) + pos_r = fx.Int64(fx.ptr_load(pos_ptr + fx.Int64(qrl))) + gmax = snk + lv, vsl = [], [] + for k in fx.range_constexpr(S): + vs = (tps * fx.Int32(k)) < ntiles + vsl.append(vs) + lv.append(vs.select(lsv[k], ninf)) + gmax = gmax.maximumf(lv[k]) + gl = lv[0] + for k in fx.range_constexpr(1, S): + gl = gl.maximumf(lv[k]) + empty_r = gl == ninf + den = exp2(snk - gmax) + wts = [] + for k in fx.range_constexpr(S): + w_ = exp2(lv[k] - gmax) + den = den + w_ + wts.append(w_) + fsc = empty_r.select(qnan, rcp(den)) + pb = part_base + ( + fx.Int64(group) * fx.Int64(S * ROWS * DV) + + fx.Int64(wv * fx.Int32(64 * 256) + lane_ld * fx.Int32(4)) + ) * fx.Int64(PEB) + zv4 = fx.Vector.filled(4, 0.0, fx.Float32) + for vq in fx.range_constexpr(4): + if fx.Int32(vq % S) == (j & fx.Int32(S - 1)): + accm = [zv4] * 16 + for k in fx.range_constexpr(S): + mine = own and k == vq % S + pk = pb + fx.Int64(k * ROWS * DV * PEB) + for gg in fx.range_constexpr(16): + d_, q_ = 4 * vq + gg // 4, gg % 4 + if fx.const_expr(mine): + v_ = fx.Vector( + rq_part_val(own_acc, ipm, d_, q_) + ) + else: + if fx.const_expr(q_ == 0): + pbase = pin_v64( + pk + fx.Int64(4 * d_ * 256 * PEB) + ) + a_ = pbase + fx.Int64(q_ * 256 * PEB) + v_ = fx.Vector(gload(a_, 4)) + accm[gg] = accm[gg] + vsl[k].select( + v_ * wts[k], zv4 + ) + + def val(k_, q_, accm=accm): + return accm[4 * k_ + q_] * fsc + + emit_rows(val, wv, fx.Int32(4 * vq), 4, empty_r, pos_r) + + if fx.const_expr(RQ or RD): + v4i32 = fx.Vector.make_type(4, fx.Int32) + v2i32 = fx.Vector.make_type(2, fx.Int32) + v16f32 = fx.Vector.make_type(16, fx.Float32) + hq = lane // fx.Int32(32) + r32 = lane % fx.Int32(32) + _dom = '#llvm.alias_scope_domain, description = "rqlds">' + _dma_scope = _ir.Attribute.parse( + f"#llvm.alias_scope, domain = {_dom}>" + ) + DMA_AS = _ir.ArrayAttr.get([_dma_scope]) + KPD = 2 + PVD = 3 + + def fence(): + fx.rocdl.sched_barrier(0) + + def early_tickets(): + tkv = zero_i + if wave == zero_i: + tkv = ld_agent_i32(tk_lane, volatile=False) + return tkv + + def lds_barrier(): + _llvm_d.InlineAsmOp( + None, + [], + "s_waitcnt lgkmcnt(0)\n s_barrier", + "~{memory}", + has_side_effects=True, + ) + + def anchor(x, cls): + return fx.Vector( + _llvm_d.InlineAsmOp( + x.ir_value().type, + [x.ir_value()], + "; " + cls + " $0", + f"={cls},0", + ).res + ) + + def lds_rd(off, ty, align): + return fx.Vector( + _llvm_d.LoadOp( + ty, + fx.to_llvm_ptr(p_rq + off), + alignment=align, + noalias_scopes=DMA_AS, + ).result + ) + + def lds_rd16(off): + return lds_rd(off, v4i32, 16) + + def lds_wr16(off, val): + _llvm_d.StoreOp( + _arith.unwrap(val), + fx.to_llvm_ptr(p_rq + off), + alignment=16, + noalias_scopes=DMA_AS, + ) + + def lds_tr(off): + return fx.Vector( + fx.rocdl.ds_read_tr16_b64( + v4bf16, fx.to_llvm_ptr(p_rq + off), noalias_scopes=DMA_AS + ).result + ) + + def dma16(gaddr, off, nt=False): + _rocdl_d.global_load_lds( + fx.to_llvm_ptr(cu._to_ptr_global(gaddr)), + fx.to_llvm_ptr(p_rq + off), + 16, + 0, + aux=_ir.IntegerAttr.get(T.i32, CPOL_NT) if nt else None, + alias_scopes=DMA_AS, + ) + + def kslot(c, k): + return fx.Int32(16) * ( + fx.Int32(32) * c + (k ^ ((c & fx.Int32(3)) * fx.Int32(4))) + ) + + def kf_base(t): + return _rem_const(t, NKF) * fx.Int32(RQ_KF) + + def kr_base(t): + return fx.Int32(RQ_KR_OFF) + _rem_const(t, NKR) * fx.Int32(RQ_KR) + + def v_base(t): + return fx.Int32(RQ_V_OFF) + (t & fx.Int32(NVB_RQ - 1)) * fx.Int32( + RQ_V + ) + + def kpx(k): + return (k >> fx.Int32(1)) & fx.Int32(7) + + def kfo(c, k): + return fx.Int32(16) * ( + (c >> fx.Int32(3)) * fx.Int32(256) + + k * fx.Int32(8) + + ((c & fx.Int32(7)) ^ kpx(k)) + ) + + def run_mixed(main, side, every=1): + side = list(side) if side is not None else [] + for k, f in enumerate(main, 1): + f() + if side and k % every == 0: + side.pop(0)() + fence() + for f in side: + f() + fence() + + def run_gen(steps): + for f in steps: + f() + fence() + + # inline asm keeps the accumulator in AGPRs and the operands in VGPRs. The backend's + # hazard recognizer does not see it: `fresh` (operand just written by VALU) needs 2 + # wait states, `last` the MFMA -> VALU read of the accumulator + def pv_mfma(va, pb, acc, last, fresh=False): + tail = "\n s_nop 7\n s_nop 7\n s_nop 3" if last else "" + return fx.Vector( + _llvm_d.InlineAsmOp( + v16f32, + [va.ir_value(), pb.ir_value(), acc.ir_value()], + ("s_nop 1\n " if fresh else "") + + "v_mfma_f32_32x32x16_bf16 $0, $1, $2, $3" + + tail, + "=a,v,v,0", + ).res + ) + + def pmax32(x): + xi = x.bitcast(fx.Int32) + return plswap( + xi, + 32, + lambda a, b: fx.Int32( + a.bitcast(fx.Float32) + .maximumf(b.bitcast(fx.Float32)) + .bitcast(fx.Int32) + ), + ).bitcast(fx.Float32) + + if fx.const_expr(RQ): + rq_vw = [wv * fx.Int32(NROLE) + fx.Int32(p) for p in range(NROLE)] + kd_fs = [vw * fx.Int32(8) + lane // fx.Int32(8) for vw in rq_vw] + kd_rs = [ + r32 ^ (((vw * fx.Int32(2) + hq) & fx.Int32(3)) * fx.Int32(4)) + for vw in rq_vw + ] + kf_pieces = [(lane % fx.Int32(8)) ^ kpx(kd) for kd in kd_fs] + rq_keys = [kk for p in range(NROLE) for kk in (kd_fs[p], kd_rs[p])] + + def idx_tile(t): + rows = [] + for kk in rq_keys: + kidx = t * fx.Int32(KTL) + kk + ok = (kidx < kvlen) & (t < t1) + rows.append(ld_idx(ok.select(kidx, zero_i))) + return rows + + def ix_off(t, vw): + return fx.Int32(RQ_IX_OFF) + ( + _rem_const(t, NIX) * fx.Int32(4) + vw + ) * fx.Int32(512) + + def dma_idx(t): + for p in fx.range_constexpr(NROLE): + for u, kk in enumerate((kd_fs[p], kd_rs[p])): + kidx = t * fx.Int32(KTL) + kk + ok = (kidx < kvlen) & (t < t1) + ga = ibase + fx.Int64(ok.select(kidx, zero_i)) * fx.Int64(4) + _rocdl_d.global_load_lds( + fx.to_llvm_ptr(cu._to_ptr_global(ga)), + fx.to_llvm_ptr( + p_rq + + ( + ix_off(t, rq_vw[p]) + + fx.Int32(256 * u) + + lane * fx.Int32(4) + ) + ), + 4, + 0, + alias_scopes=DMA_AS, + ) + + def idx_lds(t): + return [ + fx.Int32( + lds_rd( + ix_off(t, rq_vw[p]) + + fx.Int32(256 * u) + + lane * fx.Int32(4), + fx.Vector.make_type(1, fx.Int32), + 4, + )[0] + ) + for p in range(NROLE) + for u in range(2) + ] + + def dma_tile(t, rows): + for p in fx.range_constexpr(NROLE): + vw = rq_vw[p] + rf = fx.Int64(row_ok(rows[2 * p])) + rr = fx.Int64(row_ok(rows[2 * p + 1])) + gk = ( + fx.Int64(fx.ptrtoint(kv_ptr)) + + rf * fx.Int64(ROWB) + + fx.Int64(kf_pieces[p] * fx.Int32(16)) + ) + gr = ( + fx.Int64(fx.ptrtoint(kvr_ptr)) + + rr * fx.Int64(2 * ROPE) + + fx.Int64((vw * fx.Int32(2) + hq) * fx.Int32(16)) + ) + kfb = kf_base(t) + vw * fx.Int32(1024) + for cg in fx.range_constexpr(4): + dma16(gk + fx.Int64(128 * cg), kfb + fx.Int32(4096 * cg)) + dma16(gr, kr_base(t) + vw * fx.Int32(1024)) + + qrow_l = rb_qrow[0] + head_l = rb_h0[0] + r32 + if fx.const_expr(BOUNDS): + rq_bnd = rb_bnd[0] + qb = (fx.Int64(qrow_l) * fx.Int64(H) + fx.Int64(head_l)) * fx.Int64( + ROWB + ) + qop = [] + for st in fx.range_constexpr(7): + o_ = qb + fx.Int64(64 * st) + fx.Int64(hq * fx.Int32(32)) + lo = load16(q_ptr, o_) + hi = load16(q_ptr, o_ + fx.Int64(16)) + qop.append(join8(lo, hi)) + qsc = load16(q_ptr, qb + fx.Int64(NOPE)) + qrb = (fx.Int64(qrow_l) * fx.Int64(H) + fx.Int64(head_l)) * fx.Int64( + ROPE + ) + qrope = [ + fx.ptr_load( + qr_ptr + qrb + fx.Int64(fx.Int32(16 * r) + hq * fx.Int32(8)), + result_type=v8bf16, + ) + for r in range(4) + ] + + kq_o = [ + fx.Int32(16) + * ( + r32 * fx.Int32(8) + + ((fx.Int32(4 * par + jh) + hq * fx.Int32(2)) ^ kpx(r32)) + ) + for par in range(2) + for jh in range(2) + ] + ksc_o = kfo(fx.Int32(28), r32) + kr_o = [ + fx.Int32(16) + * ( + fx.Int32(32) * (fx.Int32(2 * r) + hq) + + (r32 ^ ((fx.Int32(2 * (r % 2)) + hq) * fx.Int32(4))) + ) + for r in range(2) + ] + lg = lane // fx.Int32(16) + li = lane % fx.Int32(16) + lq_a = li // fx.Int32(4) + lq_b = li % fx.Int32(4) + cG = (lg & fx.Int32(1)) * fx.Int32(2) + lq_b // fx.Int32(2) + swG = cG * fx.Int32(4) + pv_o = [ + fx.Int32(16) + * ( + fx.Int32(32) * cG + + ((fx.Int32(8 * qq) + hq * fx.Int32(4)) ^ swG) + + lq_a + ) + + (lq_b & fx.Int32(1)) * fx.Int32(8) + for qq in range(2) + ] + dq_k = tid % fx.Int32(32) + dq_src, dq_dst = [], [] + for p in fx.range_constexpr(NROLE): + dq_g = (tid + fx.Int32(NT * p)) // fx.Int32(32) + dq_h = (dq_k // fx.Int32(16)) ^ (dq_g & fx.Int32(1)) + dq_cf0 = dq_g // fx.Int32(2) + dq_srcp = [ + kfo(dq_cf0 + fx.Int32(4 * par), dq_k) + dq_h * fx.Int32(8) + for par in range(2) + ] + dq_dst0 = kslot(dq_cf0 * fx.Int32(2) + dq_h, dq_k) + dq_src += [ + dq_srcp[r % 2] + fx.Int32(4096 * (r // 2)) for r in range(7) + ] + dq_dst += [dq_dst0 + fx.Int32(4096 * r) for r in range(7)] + dq_sc = kfo(fx.Int32(28), dq_k) + NDQ = 7 * NROLE + + def dequant_steps(t): + st_ = {} + + def s0(): + st_["kfb"] = kf_base(t) + st_["vb"] = v_base(t) + st_["scd"] = lds_rd16(st_["kfb"] + dq_sc) + st_["src"] = [ + lds_rd(st_["kfb"] + dq_src[r], v2i32, 8) for r in range(2) + ] + + def item(ri): + r = ri % 7 + + def f(): + src = st_["src"] + if fx.const_expr(ri + 2 < NDQ): + src.append( + lds_rd(st_["kfb"] + dq_src[ri + 2], v2i32, 8) + ) + e = ( + fx.Int32(st_["scd"][r // 2]) >> fx.Int32(16 * (r % 2)) + ) & fx.Int32(0xFF) + scl = (e << fx.Int32(23)).bitcast(fx.Float32) + wds = [] + for wd in fx.range_constexpr(2): + for sel in fx.range_constexpr(2): + pr = fx.Vector( + fx.rocdl.cvt_scalef32_pk_bf16_fp8( + v2bf16, + fx.Int32(src[ri][wd]).ir_value(), + scl.ir_value(), + bool(sel), + ) + ) + wds.append(pr.bitcast(fx.Int32)[0]) + lds_wr16( + st_["vb"] + dq_dst[ri], + fx.Vector.from_elements(wds, fx.Int32), + ) + + return f + + return [s0] + [item(ri) for ri in range(NDQ)] + + def k_op(kfb, krb, st): + if fx.const_expr(st < 7): + o_ = fx.Int32(4096 * (st // 2)) + lo = lds_rd16(kfb + kq_o[2 * (st % 2)] + o_) + hi = lds_rd16(kfb + kq_o[2 * (st % 2) + 1] + o_) + return join8(lo, hi) + r = st - 7 + return lds_rd16( + krb + kr_o[r % 2] + fx.Int32(1024 * (r // 2) * 2) + ).bitcast(fx.BFloat16) + + def qk_mfma(st, op, ksc, sc): + if fx.const_expr(st < 7): + return fx.Vector( + fx.rocdl.mfma_scale_f32_32x32x64_f8f6f4( + v16f32, + [ + op, + qop[st], + sc, + 0, + 0, + 2 * (st % 2), + fx.Int32(ksc[st // 2]), + 2 * (st % 2), + fx.Int32(qsc[st // 2]), + ], + ) + ) + return fx.Vector( + fx.rocdl.mfma_f32_32x32x16_bf16(v16f32, [op, qrope[st - 7], sc]) + ) + + def qk_steps(t, out): + st_ = {} + + def step(st): + def f(): + if fx.const_expr(st == 0): + st_["kfb"] = kf_base(t) + st_["krb"] = kr_base(t) + st_["ksc"] = lds_rd16(st_["kfb"] + ksc_o) + st_["sc"] = fx.Vector.filled(16, 0.0, fx.Float32) + st_["ops"] = [ + k_op(st_["kfb"], st_["krb"], s2) + for s2 in range(KPD) + ] + ops = st_["ops"] + if fx.const_expr(st + KPD < 11): + ops.append(k_op(st_["kfb"], st_["krb"], st + KPD)) + st_["sc"] = qk_mfma(st, ops[st], st_["ksc"], st_["sc"]) + if fx.const_expr(st == 10): + out.append(anchor(st_["sc"], "v")) + + return f + + return [step(st) for st in range(11)] + + def softmax_steps(t, sc, m_run, l_run, masked, out): + st_ = {} + + def s_scale(): + tbase = t * fx.Int32(KTL) + ss = [fx.Float32(sc[r]) * scale_log2e for r in range(16)] + if fx.const_expr(masked): + lim = kvlen - tbase - hq * fx.Int32(4) + ss = [ + (fx.Int32(8 * (r // 4) + r % 4) < lim).select( + ss[r], ninf + ) + for r in range(16) + ] + if fx.const_expr(BOUNDS): + kb = tbase + hq * fx.Int32(4) + a0, a1, b0, b1 = rq_bnd + la0 = a0 - kb + la1 = (a1 > a0).select(a1 - a0, zero_i) + lb0 = b0 - kb + lb1 = (b1 > b0).select(b1 - b0, zero_i) + + def bmask(ss=ss, la0=la0, la1=la1, lb0=lb0, lb1=lb1): + return [ + ( + _ult(fx.Int32(8 * (r // 4) + r % 4) - la0, la1) + | _ult( + fx.Int32(8 * (r // 4) + r % 4) - lb0, lb1 + ) + ).select(ss[r], ninf) + for r in range(16) + ] + + inside = ((la0 <= zero_i) & (a1 - kb >= fx.Int32(28))) | ( + (lb0 <= zero_i) & (b1 - kb >= fx.Int32(28)) + ) + nmask = fx.Int64( + fx.rocdl.ballot(T.i64, (inside == false_).ir_value()) + ) + if nmask != fx.Int64(0): + ss = bmask() + st_["ss"] = ss + + def s_max(): + ss = st_["ss"] + mx = ss[0] + for r in fx.range_constexpr(1, 16): + mx = mx.maximumf(ss[r]) + st_["mx"] = mx + + def s_m(): + mx = pmax32(st_["mx"]) + mn = (mx > m_run + fx.Float32(8.0)).select(mx, m_run) + st_["mn"] = mn + st_["m_safe"] = (mn == ninf).select(fx.Float32(0.0), mn) + st_["alpha"] = exp2(m_run - st_["m_safe"]) + st_["pf"] = [] + + def s_exp(q4): + def f(): + st_["pf"] += [ + exp2(st_["ss"][4 * q4 + i] - st_["m_safe"]) + for i in range(4) + ] + + return f + + def s_sum(): + pf = st_["pf"] + ps0 = (pf[0] + pf[1]) + (pf[2] + pf[3]) + ps1 = (pf[4] + pf[5]) + (pf[6] + pf[7]) + ps2 = (pf[8] + pf[9]) + (pf[10] + pf[11]) + ps3 = (pf[12] + pf[13]) + (pf[14] + pf[15]) + st_["ps"] = (ps0 + ps1) + (ps2 + ps3) + + def s_pack(): + pf = st_["pf"] + pops = [ + fx.Vector.from_elements( + pf[8 * ks : 8 * ks + 8], fx.Float32 + ).to(fx.BFloat16) + for ks in range(2) + ] + out.extend( + [ + pops, + st_["mn"], + l_run * st_["alpha"] + st_["ps"], + st_["alpha"], + ] + ) + + return ( + [s_scale, s_max, s_m] + + [s_exp(q4) for q4 in range(4)] + + [s_sum, s_pack] + ) + + def v_op(vb0, vb1, kb0, kb1, j, ks): + b0, b1 = (vb0, vb1) if j < 14 else (kb0, kb1) + o_ = 2048 * (j if j < 14 else j - 14) + 256 * ks + h0_ = lds_tr(b0 + fx.Int32(o_)) + h1_ = lds_tr(b1 + fx.Int32(o_)) + return fx.Vector.from_elements( + [h0_[i] for i in range(4)] + [h1_[i] for i in range(4)], + fx.BFloat16, + ) + + def pv_steps(t, pops, accs, out): + st_ = {} + steps = [(j, ks) for ks in range(2) for j in range(16)] + + def step(n): + def f(): + if fx.const_expr(n == 0): + vb = v_base(t) + krb = kr_base(t) + st_["b"] = ( + vb + pv_o[0], + vb + pv_o[1], + krb + pv_o[0], + krb + pv_o[1], + ) + st_["ops"] = [ + v_op(*st_["b"], *steps[k]) for k in range(PVD) + ] + st_["acc"] = list(accs) + ops = st_["ops"] + if fx.const_expr(n + PVD < 32): + ops.append(v_op(*st_["b"], *steps[n + PVD])) + j, ks = steps[n] + acc = st_["acc"] + acc[j] = pv_mfma( + ops[n], pops[ks], acc[j], n == 31, n in (0, 16) + ) + if fx.const_expr(n == 31): + out.extend(acc) + + return f + + return [step(n) for n in range(32)] + + def rescale_all(accs, alpha): + out = [] + for j in fx.range_constexpr(NACC): + els = [] + st2 = _llvm_d.StructType.get_literal([T.f32, T.f32]) + for i in fx.range_constexpr(16): + r_ = _llvm_d.InlineAsmOp( + st2, + [ + fx.Float32(accs[j][i]).ir_value(), + alpha.ir_value(), + ], + "v_accvgpr_read_b32 $1, $0\n" + " s_nop 1\n" + " v_mul_f32 $1, $1, $3\n" + " v_accvgpr_write_b32 $0, $1", + "=a,=&v,0,v", + ).res + els.append(fx.Float32(_llvm_d.extractvalue(T.f32, r_, [0]))) + out.append(fx.Vector.from_elements(els, fx.Float32)) + return out + + def rescale(accs, alpha, m_run): + ne = (alpha != fx.Float32(1.0)) & (m_run != ninf) + need = fx.Int64(fx.rocdl.ballot(T.i64, ne.ir_value())) != fx.Int64( + 0 + ) + if need: + accs = rescale_all(accs, alpha) + return accs + + idx0 = [idx_tile(t0 + fx.Int32(j)) for j in range(DD)] + VG = 7 * NROLE + for j in fx.range_constexpr(DD): + dma_idx(t0 + fx.Int32(DD - 1 + j)) + dma_tile(t0 + fx.Int32(j), idx0[j]) + if fx.const_expr(MERGED): + slot, myx, tk_lane = protocol_entry() + NACC = 16 + acc0 = fx.Vector.filled(16, 0.0, fx.Float32) + acc0s = [ + _llvm_d.InlineAsmOp( + v16f32, [acc0.ir_value()], "; acc $0", "=a,0" + ).res + for _ in range(NACC) + ] + nits = t1 - t0 + if fx.const_expr(NVB_RQ == 1): + fx.rocdl.s_waitcnt(vmcnt=VG * (DD - 1)) + lds_barrier() + u_sc0l = [] + run_mixed(qk_steps(t0, u_sc0l), dequant_steps(t0)) + fx.rocdl.s_waitcnt(vmcnt=VG * (DD - 2)) + lds_barrier() + u_init = [ + ninf.ir_value(), + fx.Float32(0.0).ir_value(), + u_sc0l[0].ir_value(), + ] + acc0s + u_nmain = (nits > fx.Int32(1)).select(nits - fx.Int32(1), zero_i) + u_results = u_init + for u_it, u_state in range( + zero_i, u_nmain, fx.Int32(1), init=u_init + ): + u_m_run = fx.Float32(u_state[0]) + u_l_run = fx.Float32(u_state[1]) + u_sc = fx.Vector(u_state[2]) + u_accs = [fx.Vector(u_state[3 + j]) for j in range(NACC)] + u_tt = t0 + fx.Int32(u_it) + u_i_n = idx_lds(u_tt + fx.Int32(DD)) + dma_idx(u_tt + fx.Int32(2 * DD - 1)) + dma_tile(u_tt + fx.Int32(DD), u_i_n) + fence() + u_so = [] + run_gen(softmax_steps(u_tt, u_sc, u_m_run, u_l_run, True, u_so)) + u_pops, u_m_new, u_l_new, u_alpha = u_so + u_accs = rescale(u_accs, u_alpha, u_m_run) + fence() + u_po = [] + run_mixed(pv_steps(u_tt, u_pops, u_accs, u_po), None) + lds_barrier() + u_qo = [] + run_mixed( + qk_steps(u_tt + fx.Int32(1), u_qo), + dequant_steps(u_tt + fx.Int32(1)), + ) + fx.rocdl.s_waitcnt(vmcnt=VG * (DD - 2)) + lds_barrier() + u_results = yield ( + [ + u_m_new.ir_value(), + u_l_new.ir_value(), + u_qo[0].ir_value(), + ] + + [x.ir_value() for x in u_po] + ) + m_l = fx.Float32(u_results[0]) + l_l = fx.Float32(u_results[1]) + acc_l = [fx.Vector(u_results[3 + j]) for j in range(NACC)] + if nits > zero_i: + u_tl = t1 - fx.Int32(1) + u_so2 = [] + run_gen( + softmax_steps( + u_tl, fx.Vector(u_results[2]), m_l, l_l, True, u_so2 + ) + ) + u_pops2, u_m_l2, u_l_l2, u_alpha2 = u_so2 + u_acc_l2 = rescale(acc_l, u_alpha2, m_l) + u_po2 = [] + run_mixed(pv_steps(u_tl, u_pops2, u_acc_l2, u_po2), None) + m_l = u_m_l2 + l_l = u_l_l2 + acc_l = u_po2 + else: + fx.rocdl.s_waitcnt(vmcnt=VG * (DD - 1)) + lds_barrier() + run_gen(dequant_steps(t0)) + fx.rocdl.s_waitcnt(vmcnt=VG * (DD - 2)) + lds_barrier() + sc0l = [] + run_gen(qk_steps(t0, sc0l)) + init = [ + ninf.ir_value(), + fx.Float32(0.0).ir_value(), + sc0l[0].ir_value(), + ] + init += acc0s + nmain = (nits > fx.Int32(1)).select(nits - fx.Int32(1), zero_i) + results = init + for it, state in range(zero_i, nmain, fx.Int32(1), init=init): + m_run = fx.Float32(state[0]) + l_run = fx.Float32(state[1]) + sc_cur = fx.Vector(state[2]) + accs = [fx.Vector(state[3 + j]) for j in range(NACC)] + tt = t0 + fx.Int32(it) + i3 = idx_lds(tt + fx.Int32(DD)) + dma_idx(tt + fx.Int32(2 * DD - 1)) + dma_tile(tt + fx.Int32(DD), i3) + fence() + qo, so = [], [] + run_mixed( + qk_steps(tt + fx.Int32(1), qo), + softmax_steps(tt, sc_cur, m_run, l_run, False, so), + ) + pops, m_new, l_new, alpha = so + accs = rescale(accs, alpha, m_run) + fence() + po = [] + run_mixed( + pv_steps(tt, pops, accs, po), + dequant_steps(tt + fx.Int32(1)), + every=4 // NROLE, + ) + fx.rocdl.s_waitcnt(vmcnt=VG * (DD - 2)) + lds_barrier() + results = yield ( + [m_new.ir_value(), l_new.ir_value(), qo[0].ir_value()] + + [x.ir_value() for x in po] + ) + if fx.const_expr(OWN): + tkv_own = early_tickets() + m_l = fx.Float32(results[0]) + l_l = fx.Float32(results[1]) + acc_l = [fx.Vector(results[3 + j]) for j in range(NACC)] + if nits > zero_i: + tl = t1 - fx.Int32(1) + so = [] + run_gen( + softmax_steps(tl, fx.Vector(results[2]), m_l, l_l, True, so) + ) + pops, m_l2, l_l2, alpha = so + acc_l2 = rescale(acc_l, alpha, m_l) + po = [] + run_mixed(pv_steps(tl, pops, acc_l2, po), None) + m_l = m_l2 + l_l = l_l2 + acc_l = po + fx.rocdl.s_waitcnt(vmcnt=0) + m_fin = [m_l] + l_fin = [l_l + l_l.shuffle_xor(fx.Int32(32), fx.Int32(WAVE))] + if fx.const_expr(MERGED and RQM): + own_ip, own_lse = rq_partials(m_fin[0], l_fin[0]) + own_acc = acc_l + acc_fin = [ + [ + fx.Vector.from_elements( + [acc_l[n // 4][4 * (n % 4) + i] for i in range(4)], + fx.Float32, + ) + for n in range(64) + ] + ] + elif fx.const_expr(RD): + SPEC = RD_NQW < 4 and not Q16 + if fx.const_expr(SPEC): + dma_wave = wv >= fx.Int32(2) + dd_ = wv - fx.Int32(2) + kd_a = dd_ * fx.Int32(8) + lane // fx.Int32(8) + kd_b = kd_a + fx.Int32(16) + kd_r = r32 ^ ( + ((dd_ * fx.Int32(2) + hq) & fx.Int32(3)) * fx.Int32(4) + ) + kd_list = (kd_a, kd_b, kd_r) + NVM = 13 + else: + dma_wave = true_ + kd_f = wv * fx.Int32(8) + lane // fx.Int32(8) + kd_r = r32 ^ (((wv * fx.Int32(2) + hq) & fx.Int32(3)) * fx.Int32(4)) + kd_list = (kd_f, kd_r) + NVM = 7 + NKD = len(kd_list) + + def idx_tile(t): + rows = [] + for kk in kd_list: + kidx = t * fx.Int32(KTL) + kk + ok = (kidx < kvlen) & (t < t1) + rows.append(ld_idx(ok.select(kidx, zero_i))) + return rows + + def ix_off(t): + return fx.Int32(RQ_IX_OFF) + ( + _rem_const(t, NIX) * fx.Int32(2) + dd_ + ) * fx.Int32(256 * NKD) + + def dma_idx(t): + for u, kk in enumerate(kd_list): + kidx = t * fx.Int32(KTL) + kk + ok = (kidx < kvlen) & (t < t1) + ga = ibase + fx.Int64(ok.select(kidx, zero_i)) * fx.Int64(4) + _rocdl_d.global_load_lds( + fx.to_llvm_ptr(cu._to_ptr_global(ga)), + fx.to_llvm_ptr( + p_rq + + (ix_off(t) + fx.Int32(256 * u) + lane * fx.Int32(4)) + ), + 4, + 0, + alias_scopes=DMA_AS, + ) + + def idx_lds(t): + return [ + fx.Int32( + lds_rd( + ix_off(t) + fx.Int32(256 * u) + lane * fx.Int32(4), + fx.Vector.make_type(1, fx.Int32), + 4, + )[0] + ) + for u in range(NKD) + ] + + def dma_kf_block(t, row, kd, kb): + r_ = fx.Int64(row_ok(row)) + gk = ( + fx.Int64(fx.ptrtoint(kv_ptr)) + + r_ * fx.Int64(ROWB) + + fx.Int64(((lane % fx.Int32(8)) ^ kpx(kd)) * fx.Int32(16)) + ) + kfb = kf_base(t) + kb * fx.Int32(1024) + for cg in fx.range_constexpr(4): + dma16(gk + fx.Int64(128 * cg), kfb + fx.Int32(4096 * cg)) + + def dma_kr_item(t, row, p): + rr = fx.Int64(row_ok(row)) + gr = ( + fx.Int64(fx.ptrtoint(kvr_ptr)) + + rr * fx.Int64(2 * ROPE) + + fx.Int64((p * fx.Int32(2) + hq) * fx.Int32(16)) + ) + dma16(gr, kr_base(t) + p * fx.Int32(1024)) + + def dma_tile(t, rows): + dma_kf_block(t, rows[0], kd_a, dd_) + dma_kf_block(t, rows[1], kd_b, dd_ + fx.Int32(2)) + dma_kr_item(t, rows[2], dd_) + dma_kr_item(t, rows[2], dd_ + fx.Int32(2)) + + if fx.const_expr(Q16): + q_lim = smax_i32( + smin_i32(kvlen, t1 * fx.Int32(KTL)) - fx.Int32(1), zero_i + ) + kf_lane = fx.Int64(fx.ptrtoint(kv_ptr)) + fx.Int64( + ((lane % fx.Int32(8)) ^ kpx(kd_f)) * fx.Int32(16) + ) + kr_lane = fx.Int64(fx.ptrtoint(kvr_ptr)) + fx.Int64( + (wv * fx.Int32(2) + hq) * fx.Int32(16) + ) + + def ix_clamp(t, kk): + kidx = t * fx.Int32(KTL) + kk + return (kidx < q_lim).select(kidx, q_lim) + + def row_u(raw): + return fx.Int64(_ult(raw, num_rows).select(raw, zero_i)) + + dq_k = lane % fx.Int32(32) + dq_h = (dq_k // fx.Int32(16)) ^ hq + dq_sc = kfo(fx.Int32(28), dq_k) + + dqa_src = [ + kfo(wv + fx.Int32(4 * par), dq_k) + dq_h * fx.Int32(8) + for par in range(2) + ] + dqa_dst = kslot(wv * fx.Int32(2) + dq_h, dq_k) + + def dequant_steps(t): + st_ = {} + + def s0(): + st_["kfb"] = kf_base(t) + st_["vb"] = v_base(t) + st_["scd"] = lds_rd16(st_["kfb"] + dq_sc) + st_["src"] = [ + lds_rd( + st_["kfb"] + dqa_src[r % 2] + fx.Int32(4096 * (r // 2)), + v2i32, + 8, + ) + for r in range(2) + ] + + def item(r): + def f(): + src = st_["src"] + if fx.const_expr(r + 2 < 7): + r2 = r + 2 + src.append( + lds_rd( + st_["kfb"] + + dqa_src[r2 % 2] + + fx.Int32(4096 * (r2 // 2)), + v2i32, + 8, + ) + ) + e = ( + fx.Int32(st_["scd"][r // 2]) >> fx.Int32(16 * (r % 2)) + ) & fx.Int32(0xFF) + scl = (e << fx.Int32(23)).bitcast(fx.Float32) + wds = [] + for wd in fx.range_constexpr(2): + for sel in fx.range_constexpr(2): + pr = fx.Vector( + fx.rocdl.cvt_scalef32_pk_bf16_fp8( + v2bf16, + fx.Int32(src[r][wd]).ir_value(), + scl.ir_value(), + bool(sel), + ) + ) + wds.append(pr.bitcast(fx.Int32)[0]) + lds_wr16( + st_["vb"] + dqa_dst + fx.Int32(4096 * r), + fx.Vector.from_elements(wds, fx.Int32), + ) + + return f + + return [s0] + [item(r) for r in range(7)] + + NQW = RD_NQW + NPB = ROWS // 32 + NACC = 4 * NPB + qk_wave = wv < fx.Int32(NQW) + PP = 80 + + def p_off(row, slot16): + return ( + fx.Int32(RD_P_OFF) + + row * fx.Int32(PP) + + (slot16 ^ ((row >> fx.Int32(4)) & fx.Int32(1))) * fx.Int32(16) + ) + + if fx.const_expr(Q16): + is_qk = wv >= fx.Int32(4) + wq = wv & fx.Int32(3) + qk_wave = is_qk + + def load_q16(): + q_jb = jb0 + wq + q_qp = _div_const(q_jb, HB) + qrow_l = q0 + (q_qp < qlen).select(q_qp, qlen - fx.Int32(1)) + head_l = _rem_const(q_jb, HB) * fx.Int32(16) + m + qb = ( + fx.Int64(qrow_l) * fx.Int64(H) + fx.Int64(head_l) + ) * fx.Int64(ROWB) + qop = [] + for cc in fx.range_constexpr(4): + lo = load16( + q_ptr, qb + fx.Int64(128 * cc) + fx.Int64(gsh * 2) + ) + if fx.const_expr(cc < 3): + hi = load16( + q_ptr, + qb + fx.Int64(128 * cc + 64) + fx.Int64(gsh * 2), + ) + else: + hi = zero4 + qop.append(join8(lo, hi)) + qsc16 = load16(q_ptr, qb + fx.Int64(NOPE)) + qscl16 = [_scl(qsc16[cc], cc) for cc in range(4)] + qrb = ( + fx.Int64(qrow_l) * fx.Int64(H) + fx.Int64(head_l) + ) * fx.Int64(ROPE) + qrope = [ + fx.ptr_load( + qr_ptr + qrb + fx.Int64(gsh + fx.Int32(32 * j)), + result_type=v8bf16, + ) + for j in range(2) + ] + return qop, qscl16, qrope + + def prefetch_next(): + nb = fx.Int32(fx.block_idx.x) + fx.Int32(cfg.PFN) + nblk = ((nseq + fx.Int32(7)) >> fx.Int32(3)) * fx.Int32( + 8 * NRG * S + ) + + def warm(sq, rgn): + if fx.const_expr(EPI_Q and MSQ == 1): + q0n, qln = sq, fx.Int32(1) + else: + q0n = _sload(qo_ptr, sq) + qln = _sload(qo_ptr, sq + fx.Int32(1)) - q0n + kv0n = _sload(indptr_ptr, sq) + kvln = _sload(indptr_ptr, sq + fx.Int32(1)) - kv0n + gl = wq * fx.Int32(WAVE) + lane + acc = zero_i + NQL = ROWS * 5 + for j in range((NQL + 4 * WAVE - 1) // (4 * WAVE)): + ln = gl + fx.Int32(4 * WAVE * j) + lc = (ln < fx.Int32(NQL)).select(ln, zero_i) + r = _div_const(lc, 5) + part = lc - r * fx.Int32(5) + jb = rgn * fx.Int32(RB) + (r >> fx.Int32(4)) + qp = _div_const(jb, HB) + qp = (qp < qln).select(qp, qln - fx.Int32(1)) + qrow = (qln > zero_i).select(q0n + qp, zero_i) + hd = _rem_const(jb, HB) * fx.Int32(16) + ( + r & fx.Int32(15) + ) + ri = fx.Int64(qrow) * fx.Int64(H) + fx.Int64(hd) + qa = (part < fx.Int32(4)).select( + fx.Int64(fx.ptrtoint(q_ptr)) + + ri * fx.Int64(ROWB) + + fx.Int64(part) * fx.Int64(128), + fx.Int64(fx.ptrtoint(qr_ptr)) + + ri * fx.Int64(2 * ROPE), + ) + acc = acc ^ fx.Int32( + fx.generic_load(cu._to_ptr_global(qa)) + ) + ki = gl * fx.Int32(32) + kic = ((ki < kvln) & (gl < fx.Int32(8))).select(ki, zero_i) + ia = (kvln > zero_i).select( + fx.Int64(fx.ptrtoint(idx_ptr)) + + (fx.Int64(kv0n) + fx.Int64(kic)) * fx.Int64(4), + fx.Int64(fx.ptrtoint(indptr_ptr)) + + fx.Int64(sq) * fx.Int64(4), + ) + acc = acc ^ fx.Int32(fx.generic_load(cu._to_ptr_global(ia))) + if (nseq < zero_i) & (acc == fx.Int32(1)): + fx.ptr_store(fx.Float32(0.0), sink_ptr) + + sq, rgn, _ = map_bid(nb, nseq) + if (nb < nblk) & (sq < nseq): + warm(sq, rgn) + + k16 = [fx.Int32(16 * kb) + m for kb in range(2)] + kq16_o = [ + [ + fx.Int32(16) + * ( + k16[kb] * fx.Int32(8) + + ((fx.Int32(4 * hf) + g) ^ kpx(k16[kb])) + ) + for hf in range(2) + ] + for kb in range(2) + ] + ksc16_o = [kfo(fx.Int32(28), k16[kb]) for kb in range(2)] + kr16_o = [ + fx.Int32(16) + * (g * fx.Int32(32) + (k16[kb] ^ (g * fx.Int32(4)))) + for kb in range(2) + ] + sm_row = wq * fx.Int32(16) + m + sm_first = g == zero_i + + else: + q_row_l = wv * fx.Int32(32) + r32 + q_jb = jb0 + wv * fx.Int32(2) + q_qp = _div_const(q_jb, HB) + qrow_l = q0 + (q_qp < qlen).select(q_qp, qlen - fx.Int32(1)) + head_l = _rem_const(q_jb, HB) * fx.Int32(16) + r32 + qb = (fx.Int64(qrow_l) * fx.Int64(H) + fx.Int64(head_l)) * fx.Int64( + ROWB + ) + qop = [] + for st in fx.range_constexpr(7): + o_ = qb + fx.Int64(64 * st) + fx.Int64(hq * fx.Int32(32)) + lo = load16(q_ptr, o_) + hi = load16(q_ptr, o_ + fx.Int64(16)) + qop.append(join8(lo, hi)) + qsc = load16(q_ptr, qb + fx.Int64(NOPE)) + qrb = ( + fx.Int64(qrow_l) * fx.Int64(H) + fx.Int64(head_l) + ) * fx.Int64(ROPE) + qrope = [ + fx.ptr_load( + qr_ptr + + qrb + + fx.Int64(fx.Int32(16 * r) + hq * fx.Int32(8)), + result_type=v8bf16, + ) + for r in range(4) + ] + kq_o = [ + fx.Int32(16) + * ( + r32 * fx.Int32(8) + + ((fx.Int32(4 * par + jj) + hq * fx.Int32(2)) ^ kpx(r32)) + ) + for par in range(2) + for jj in range(2) + ] + ksc_o = kfo(fx.Int32(28), r32) + kr_o = [ + fx.Int32(16) + * ( + fx.Int32(32) * (fx.Int32(2 * r) + hq) + + (r32 ^ ((fx.Int32(2 * (r % 2)) + hq) * fx.Int32(4))) + ) + for r in range(2) + ] + + def k_op(kfb, krb, st): + if fx.const_expr(st < 7): + o_ = fx.Int32(4096 * (st // 2)) + lo = lds_rd16(kfb + kq_o[2 * (st % 2)] + o_) + hi = lds_rd16(kfb + kq_o[2 * (st % 2) + 1] + o_) + return join8(lo, hi) + r = st - 7 + return lds_rd16( + krb + kr_o[r % 2] + fx.Int32(1024 * (r // 2) * 2) + ).bitcast(fx.BFloat16) + + def qk_steps(t, out): + st_ = {} + + def step(st): + def f(): + if fx.const_expr(st == 0): + st_["kfb"] = kf_base(t) + st_["krb"] = kr_base(t) + st_["ksc"] = lds_rd16(st_["kfb"] + ksc_o) + st_["sc"] = fx.Vector.filled(16, 0.0, fx.Float32) + st_["ops"] = [ + k_op(st_["kfb"], st_["krb"], s2) + for s2 in range(KPD) + ] + ops = st_["ops"] + if fx.const_expr(st + KPD < 11): + ops.append(k_op(st_["kfb"], st_["krb"], st + KPD)) + if fx.const_expr(st < 7): + st_["sc"] = fx.Vector( + fx.rocdl.mfma_scale_f32_32x32x64_f8f6f4( + v16f32, + [ + ops[st], + qop[st], + st_["sc"], + 0, + 0, + 2 * (st % 2), + fx.Int32(st_["ksc"][st // 2]), + 2 * (st % 2), + fx.Int32(qsc[st // 2]), + ], + ) + ) + else: + st_["sc"] = fx.Vector( + fx.rocdl.mfma_f32_32x32x16_bf16( + v16f32, [ops[st], qrope[st - 7], st_["sc"]] + ) + ) + if fx.const_expr(st == 10): + out.append(st_["sc"]) + + return f + + return [step(st) for st in range(11)] + + def sc_list(sc): + return [fx.Float32(sc[0][r]) for r in range(16)] + + def key_of(idx): + return fx.Int32(8 * (idx // 4) + idx % 4) + hq * fx.Int32(4) + + def row_max(x): + return pmax32(x) + + def row_sum(x): + return x + x.shuffle_xor(fx.Int32(32), fx.Int32(WAVE)) + + NSC = 16 + sm_row = q_row_l + sm_first = hq == zero_i + + def p_write(pf): + for j in fx.range_constexpr(4): + pb = fx.Vector.from_elements( + pf[4 * j : 4 * j + 4], fx.Float32 + ).to(fx.BFloat16) + _llvm_d.StoreOp( + _arith.unwrap(pb.bitcast(fx.Int32)), + fx.to_llvm_ptr( + p_rq + + (p_off(sm_row, fx.Int32(j)) + hq * fx.Int32(8)) + ), + alignment=8, + noalias_scopes=DMA_AS, + ) + + def softmax_steps(t, sc, m_run, l_run, masked, out): + st_ = {} + + def s_scale(): + tbase = t * fx.Int32(KTL) + ss = [x * scale_log2e for x in sc_list(sc)] + if fx.const_expr(masked): + lim = kvlen - tbase + ss = [ + (key_of(r) < lim).select(ss[r], ninf) + for r in range(NSC) + ] + st_["ss"] = ss + + def s_m(): + ss = st_["ss"] + mx = ss[0] + for r in fx.range_constexpr(1, NSC): + mx = mx.maximumf(ss[r]) + mx = row_max(mx) + mn = (mx > m_run + fx.Float32(8.0)).select(mx, m_run) + st_["mn"] = mn + st_["m_safe"] = (mn == ninf).select(fx.Float32(0.0), mn) + st_["alpha"] = exp2(m_run - st_["m_safe"]) + st_["pf"] = [] + + def s_exp(q4): + def f(): + st_["pf"] += [ + exp2(st_["ss"][4 * q4 + i] - st_["m_safe"]) + for i in range(4) + ] + + return f + + def s_out(): + pf = st_["pf"] + ps = pf[0] + for r in fx.range_constexpr(1, NSC): + ps = ps + pf[r] + ps = row_sum(ps) + p_write(pf) + a_acc = (m_run == ninf).select( + fx.Float32(1.0), st_["alpha"] + ) + if sm_first: + _llvm_d.StoreOp( + _arith.unwrap(a_acc), + fx.to_llvm_ptr( + p_rq + + (fx.Int32(RD_A_OFF) + sm_row * fx.Int32(4)) + ), + alignment=4, + noalias_scopes=DMA_AS, + ) + out.extend([st_["mn"], l_run * st_["alpha"] + ps]) + + return ( + [s_scale, s_m] + + [s_exp(q4) for q4 in range(NSC // 4)] + + [s_out] + ) + + lg = lane // fx.Int32(16) + li = lane % fx.Int32(16) + lq_a = li // fx.Int32(4) + lq_b = li % fx.Int32(4) + cG = (lg & fx.Int32(1)) * fx.Int32(2) + lq_b // fx.Int32(2) + swG = cG * fx.Int32(4) + pvd_o = [ + fx.Int32(16) + * ( + fx.Int32(32) * cG + + ((hq * fx.Int32(8) + fx.Int32(4 * qq)) ^ swG) + + lq_a + ) + + (lq_b & fx.Int32(1)) * fx.Int32(8) + for qq in range(2) + ] + pv_dvb = wv * fx.Int32(4) + rope_w = wv == fx.Int32(3) + + def v_opd(t, jj, ks): + vb = v_base(t) + krb = kr_base(t) + j = pv_dvb + fx.Int32(jj) + if fx.const_expr(jj < 2): + base = vb + j * fx.Int32(2048) + else: + base = rope_w.select( + krb + (j - fx.Int32(14)) * fx.Int32(2048), + vb + j * fx.Int32(2048), + ) + o_ = base + fx.Int32(256 * ks) + h0_ = lds_tr(o_ + pvd_o[0]) + h1_ = lds_tr(o_ + pvd_o[1]) + return fx.Vector.from_elements( + [h0_[i] for i in range(4)] + [h1_[i] for i in range(4)], + fx.BFloat16, + ) + + def p_opd(rb, ks): + row = fx.Int32(32 * rb) + r32 + return lds_rd16(p_off(row, fx.Int32(2 * ks) + hq)).bitcast( + fx.BFloat16 + ) + + def pv_steps(t, accs, out, last=False): + st_ = {} + steps = [ + (ks, jj, rb) + for ks in range(2) + for jj in range(4) + for rb in range(NPB) + ] + NS = len(steps) + + def ops_of(n): + ks, jj, rb = steps[n] + if fx.const_expr(rb == 0): + st_["va"][(ks, jj)] = v_opd(t, jj, ks) + if fx.const_expr(jj == 0): + st_["pb"][(ks, rb)] = p_opd(rb, ks) + + def step(n): + def f(): + if fx.const_expr(n == 0): + st_["va"] = {} + st_["pb"] = {} + st_["acc"] = list(accs) + for k0 in fx.range_constexpr(min(RD_PVD, NS)): + ops_of(k0) + if fx.const_expr(n + RD_PVD < NS): + ops_of(n + RD_PVD) + ks, jj, rb = steps[n] + ai = jj * NPB + rb + st_["acc"][ai] = pv_mfma( + st_["va"][(ks, jj)], + st_["pb"][(ks, rb)], + st_["acc"][ai], + last and n == NS - 1, + ) + if fx.const_expr(n == NS - 1): + out.extend(st_["acc"]) + + return f + + return [step(n) for n in range(NS)] + + def rescale_d(accs): + al = [ + fx.Float32( + lds_rd( + fx.Int32(RD_A_OFF) + + (fx.Int32(32 * rb) + r32) * fx.Int32(4), + fx.Vector.make_type(1, fx.Float32), + 4, + )[0] + ) + for rb in range(NPB) + ] + ne = al[0] != fx.Float32(1.0) + for rb in fx.range_constexpr(1, NPB): + ne = ne | (al[rb] != fx.Float32(1.0)) + need = fx.Int64(fx.rocdl.ballot(T.i64, ne.ir_value())) != fx.Int64( + 0 + ) + if need: + accs = [ + rescale_blk(accs[ai], al[ai % NPB]) for ai in range(NACC) + ] + return accs + + def rescale_blk(a, alpha, n=16): + st2 = _llvm_d.StructType.get_literal([T.f32, T.f32]) + els = [] + for i in fx.range_constexpr(n): + r_ = _llvm_d.InlineAsmOp( + st2, + [fx.Float32(a[i]).ir_value(), alpha.ir_value()], + "v_accvgpr_read_b32 $1, $0\n" + " s_nop 1\n" + " v_mul_f32 $1, $1, $3\n" + " v_accvgpr_write_b32 $0, $1", + "=a,=&v,0,v", + ).res + els.append(fx.Float32(_llvm_d.extractvalue(T.f32, r_, [0]))) + return fx.Vector.from_elements(els, fx.Float32) + + if fx.const_expr(Q16): + PBUF = ROWS * PP + ABUF = ROWS * 4 + + def ro(ph, n, stride, extra=0): + if isinstance(ph, int): + return (ph % n) * stride + extra + return (ph & fx.Int32(n - 1)) * fx.Int32(stride) + fx.Int32( + extra + ) + + def at(base, off): + return base + (fx.Int32(off) if isinstance(off, int) else off) + + def padd(ph, k): + return ph + k if isinstance(ph, int) else ph + fx.Int32(k) + + kr_b = [fx.Int32(RQ_KR_OFF) + kr16_o[kb] for kb in range(2)] + pw_b = ( + fx.Int32(RD_P_OFF) + + sm_row * fx.Int32(PP) + + ((g >> fx.Int32(1)) ^ (wq & fx.Int32(1))) * fx.Int32(16) + + (g & fx.Int32(1)) * fx.Int32(8) + ) + aw_b = fx.Int32(RD_A_OFF) + sm_row * fx.Int32(4) + hi_g = g >= fx.Int32(2) + + KPF = 3 + QK_ORDER = [(kb, st) for st in range(6) for kb in range(2)] + + def k_op(ph, kb, st): + if fx.const_expr(st < 4): + lo = lds_rd16( + at(kq16_o[kb][0], ro(ph, NKF, RQ_KF, 4096 * st)) + ) + hi = lds_rd16( + at( + kq16_o[kb][1], + ro(ph, NKF, RQ_KF, 4096 * st if st < 3 else 0), + ) + ) + return join8(lo, hi) + return lds_rd16( + at(kr_b[kb], ro(ph, NKR, RQ_KR, 2048 * (st - 4))) + ).bitcast(fx.BFloat16) + + def qk(q16, ph): + qop, qscl16, qrope = q16 + ksc = [ + lds_rd16(at(ksc16_o[kb], ro(ph, NKF, RQ_KF))) + for kb in range(2) + ] + ops = {} + for n in fx.range_constexpr(KPF): + ops[n] = k_op(ph, *QK_ORDER[n]) + scl = [] + for kb in fx.range_constexpr(2): + row = [] + for cc in fx.range_constexpr(4): + b = fx.Int32(ksc[kb][cc]) >> gsh + if fx.const_expr(cc == 3): + b = hi_g.select(fx.Int32(127), b) + row.append(b) + scl.append(row) + sc = [fx.Vector.filled(4, 0.0, fx.Float32) for _ in range(2)] + for n in fx.range_constexpr(len(QK_ORDER)): + if fx.const_expr(n + KPF < len(QK_ORDER)): + ops[n + KPF] = k_op(ph, *QK_ORDER[n + KPF]) + kb, st = QK_ORDER[n] + if fx.const_expr(st < 4): + sc[kb] = fx.Vector( + fx.rocdl.mfma_scale_f32_16x16x128_f8f6f4( + v4f32, + [ + ops[n], + qop[st], + sc[kb], + 0, + 0, + 0, + scl[kb][st], + 0, + qscl16[st], + ], + ) + ) + else: + sc[kb] = fx.Vector( + fx.rocdl.mfma_f32_16x16x32_bf16( + v4f32, [ops[n], qrope[st - 4], sc[kb]] + ) + ) + return sc + + def softmax(ph, sc, m_run, l_run, lim, tt, bnd): + ss = [ + fx.Float32(sc[kb][i]) * scale_log2e + for kb in range(2) + for i in range(4) + ] + if fx.const_expr(lim is not None): + ss = [ + ( + fx.Int32(16 * (r // 4) + r % 4) + g * fx.Int32(4) + < lim + ).select(ss[r], ninf) + for r in range(8) + ] + if fx.const_expr(BOUNDS): + a0, a1, b0, b1 = bnd + tb = tt * fx.Int32(KTL) + te = tb + fx.Int32(KTL) + inside = ((a0 <= tb) & (a1 >= te)) | ( + (b0 <= tb) & (b1 >= te) + ) + kl = tb + g * fx.Int32(4) + la0 = a0 - kl + lb0 = b0 - kl + wa = (a1 > a0).select(a1 - a0, zero_i) + wb = (b1 > b0).select(b1 - b0, zero_i) + + def bmask(ss=ss, la0=la0, lb0=lb0, wa=wa, wb=wb): + return [ + ( + _ult(fx.Int32(16 * (r // 4) + r % 4) - la0, wa) + | _ult( + fx.Int32(16 * (r // 4) + r % 4) - lb0, wb + ) + ).select(ss[r], ninf) + for r in range(8) + ] + + if inside == false_: + ss = bmask() + mx = ss[0] + for r in fx.range_constexpr(1, 8): + mx = mx.maximumf(ss[r]) + mx = plswap_f(plswap_f(mx, 16, _fmax), 32, _fmax) + mn = (mx > m_run + fx.Float32(8.0)).select(mx, m_run) + m_safe = (mn == ninf).select(fx.Float32(0.0), mn) + alpha = exp2(m_run - m_safe) + pf = [exp2(ss[r] - m_safe) for r in range(8)] + pb_ = ro(ph, 2, PBUF) + for kb in fx.range_constexpr(2): + pbv = fx.Vector.from_elements( + pf[4 * kb : 4 * kb + 4], fx.Float32 + ).to(fx.BFloat16) + _llvm_d.StoreOp( + _arith.unwrap(pbv.bitcast(fx.Int32)), + fx.to_llvm_ptr( + p_rq + + at( + pw_b, + ( + pb_ + 32 * kb + if isinstance(pb_, int) + else pb_ + fx.Int32(32 * kb) + ), + ) + ), + alignment=8, + noalias_scopes=DMA_AS, + ) + ps = pf[0] + for r in fx.range_constexpr(1, 8): + ps = ps + pf[r] + ps = plswap_f(plswap_f(ps, 16, _fadd), 32, _fadd) + a_acc = (m_run == ninf).select(fx.Float32(1.0), alpha) + if sm_first: + _llvm_d.StoreOp( + _arith.unwrap(a_acc), + fx.to_llvm_ptr(p_rq + at(aw_b, ro(ph, 2, ABUF))), + alignment=4, + noalias_scopes=DMA_AS, + ) + return mn, l_run * alpha + ps + + def qk_step(q16, ph, m_run, l_run, lim, tt, bnd): + sc = qk(q16, ph) + mn, ln = softmax(ph, sc, m_run, l_run, lim, tt, bnd) + lds_barrier() + return mn, ln + + vt_b = [ + fx.Int32(RQ_V_OFF) + wv * fx.Int32(8192) + pvd_o[q] + for q in range(2) + ] + vt23_b = [ + rope_w.select( + fx.Int32(RQ_KR_OFF) + pvd_o[q], + fx.Int32(RQ_V_OFF + 4096) + wv * fx.Int32(8192) + pvd_o[q], + ) + for q in range(2) + ] + prx = (r32 >> fx.Int32(4)) & fx.Int32(1) + pr_b = ( + fx.Int32(RD_P_OFF) + + r32 * fx.Int32(PP) + + (hq ^ prx) * fx.Int32(16) + ) + ar_b = fx.Int32(RD_A_OFF) + r32 * fx.Int32(4) + ixr_b = ( + fx.Int32(RQ_IX_OFF) + wv * fx.Int32(512) + lane * fx.Int32(4) + ) + hd = rope_w.select(zero_i, hq) + dqr_b = [ + fx.Int32(16) + * ( + wq * fx.Int32(256) + + r32 * fx.Int32(8) + + ((hd * fx.Int32(4) + fx.Int32(j)) ^ kpx(r32)) + ) + for j in range(4) + ] + dqs_b = kfo(fx.Int32(28), r32) + wq * fx.Int32(4) + dq_sh = fx.Int32(23) - hd * fx.Int32(16) + dqw_b = [ + fx.Int32(RQ_V_OFF) + + wq * fx.Int32(8192) + + hd * fx.Int32(4096) + + fx.Int32(16) * (r32 ^ fx.Int32(4 * c)) + for c in range(4) + ] + + def dequant_steps(ph): + kb_ = ro(ph, NKF, RQ_KF) + vb = ro(ph, 2, RQ_V) + st_ = {} + + def rd(): + st_["d"] = fx.Int32( + lds_rd( + at(dqs_b, kb_), fx.Vector.make_type(1, fx.Int32), 4 + )[0] + ) + st_["src"] = [lds_rd16(at(dqr_b[j], kb_)) for j in range(4)] + + def cv(j, e): + def f(): + if fx.const_expr(j == 0 and e == 0): + st_["scl"] = ( + (st_["d"] << dq_sh) & fx.Int32(0x7F800000) + ).bitcast(fx.Float32) + scl = st_["scl"] + wds = [] + for wd in fx.range_constexpr(2): + for sel in fx.range_constexpr(2): + pr = fx.Vector( + fx.rocdl.cvt_scalef32_pk_bf16_fp8( + v2bf16, + fx.Int32( + st_["src"][j][2 * e + wd] + ).ir_value(), + scl.ir_value(), + bool(sel), + ) + ) + wds.append(pr.bitcast(fx.Int32)[0]) + c = (2 * j + e) & 3 + lds_wr16( + at( + dqw_b[c], + ( + vb + 512 * (2 * j + e) + if isinstance(vb, int) + else vb + fx.Int32(512 * (2 * j + e)) + ), + ), + fx.Vector.from_elements(wds, fx.Int32), + ) + + return f + + return [rd] + [cv(j, e) for j in range(4) for e in range(2)] + + def pv_ops(ph, b23, n): + ks, jj, rb = PV_SEQ[n] + vb = ro(ph, 2, RQ_V) + pb_ = ro(ph, 2, PBUF) + va = pb = None + if fx.const_expr(rb == 0): + if fx.const_expr(jj < 2): + hs = [ + lds_tr( + at( + vt_b[q], + ( + vb + 2048 * jj + 256 * ks + if isinstance(vb, int) + else vb + fx.Int32(2048 * jj + 256 * ks) + ), + ) + ) + for q in range(2) + ] + else: + hs = [ + lds_tr( + b23[q] + fx.Int32(2048 * (jj - 2) + 256 * ks) + ) + for q in range(2) + ] + va = fx.Vector.from_elements( + [hs[0][i] for i in range(4)] + + [hs[1][i] for i in range(4)], + fx.BFloat16, + ) + if fx.const_expr(jj == 0): + pb = lds_rd16( + at( + pr_b, + ( + pb_ + 2560 * rb + 32 * ks + if isinstance(pb_, int) + else pb_ + fx.Int32(2560 * rb + 32 * ks) + ), + ) + ).bitcast(fx.BFloat16) + return va, pb + + PV_SEQ = [ + (ks, jj, rb) + for ks in range(2) + for jj in range(4) + for rb in range(NPB) + ] + + def pv_steps(ph, accs, out): + st_ = {} + NS = len(PV_SEQ) + + def fetch(n): + a_, b_ = pv_ops(ph, st_["b23"], n) + ks, jj, rb = PV_SEQ[n] + if fx.const_expr(a_ is not None): + st_["va"][(ks, jj)] = a_ + if fx.const_expr(b_ is not None): + st_["pb"][(ks, rb)] = b_ + + def step(n): + def f(): + if fx.const_expr(n == 0): + sel = rope_w.select( + fx.Int32(ro(ph, NKR, RQ_KR)), + fx.Int32(ro(ph, 2, RQ_V)), + ) + st_["b23"] = [vt23_b[q] + sel for q in range(2)] + st_["va"], st_["pb"] = {}, {} + st_["acc"] = list(accs) + for k0 in fx.range_constexpr(min(RD_PVD, NS)): + fetch(k0) + if fx.const_expr(n + RD_PVD < NS): + fetch(n + RD_PVD) + ks, jj, rb = PV_SEQ[n] + ai = jj * NPB + rb + st_["acc"][ai] = pv_mfma( + st_["va"][(ks, jj)], + st_["pb"][(ks, rb)], + st_["acc"][ai], + False, + ) + if fx.const_expr(n == NS - 1): + out.extend(st_["acc"]) + + return f + + return [step(n) for n in range(NS)] + + def al_read(ph): + ab = ro(ph, 2, ABUF) + return [ + fx.Float32( + lds_rd( + at( + ar_b, + ( + ab + 128 * rb + if isinstance(ab, int) + else ab + fx.Int32(128 * rb) + ), + ), + fx.Vector.make_type(1, fx.Float32), + 4, + )[0] + ) + for rb in range(NPB) + ] + + def rescale_al(accs, al): + ne = al[0] != fx.Float32(1.0) + for rb in fx.range_constexpr(1, NPB): + ne = ne | (al[rb] != fx.Float32(1.0)) + need = fx.Int64( + fx.rocdl.ballot(T.i64, ne.ir_value()) + ) != fx.Int64(0) + if need: + accs = [ + rescale_blk(accs[ai], al[ai % NPB]) + for ai in range(NACC) + ] + return accs + + if fx.const_expr(PV16R): + t16a = m >> fx.Int32(2) + t16b = m & fx.Int32(3) + t16c = t16b >> fx.Int32(1) + + def t16_lane(par, qq): + k = g * fx.Int32(8) + fx.Int32(4 * qq) + t16a + return fx.Int32(16) * ( + t16c * fx.Int32(32) + + (k ^ ((fx.Int32(2 * par) + t16c) * fx.Int32(4))) + ) + (t16b & fx.Int32(1)) * fx.Int32(8) + + v16_lo = [ + [ + fx.Int32(RQ_V_OFF) + + wv * fx.Int32(8192) + + t16_lane(par, qq) + for qq in range(2) + ] + for par in range(2) + ] + v16_hi = [ + [ + rope_w.select( + fx.Int32(RQ_KR_OFF) + t16_lane(par, qq), + fx.Int32(RQ_V_OFF + 4096) + + wv * fx.Int32(8192) + + t16_lane(par, qq), + ) + for qq in range(2) + ] + for par in range(2) + ] + p16_b = [ + fx.Int32(RD_P_OFF) + + m * fx.Int32(PP) + + (g ^ fx.Int32(par)) * fx.Int32(16) + for par in range(2) + ] + a16_b = fx.Int32(RD_A_OFF) + m * fx.Int32(4) + PV16_SEQ = [(blk, rb) for blk in range(8) for rb in range(RB)] + + def pv16_mfma(va, pb, acc): + return fx.Vector( + _llvm_d.InlineAsmOp( + v4f32, + [va.ir_value(), pb.ir_value(), acc.ir_value()], + "v_mfma_f32_16x16x32_bf16 $0, $1, $2, $3", + "=a,v,v,0", + ).res + ) + + def pv16_a(ph, hi, blk): + par = blk & 1 + if fx.const_expr(blk < 4): + vb = ro(ph, 2, RQ_V) + hs = [ + lds_tr( + at( + v16_lo[par][qq], + ( + vb + 1024 * blk + if isinstance(vb, int) + else vb + fx.Int32(1024 * blk) + ), + ) + ) + for qq in range(2) + ] + else: + hs = [ + lds_tr(hi[par][qq] + fx.Int32(1024 * (blk - 4))) + for qq in range(2) + ] + return fx.Vector.from_elements( + [hs[0][i] for i in range(4)] + + [hs[1][i] for i in range(4)], + fx.BFloat16, + ) + + def pv16_b(ph, rb): + pb_ = ro(ph, 2, PBUF) + return lds_rd16( + at( + p16_b[rb & 1], + ( + pb_ + 1280 * rb + if isinstance(pb_, int) + else pb_ + fx.Int32(1280 * rb) + ), + ) + ).bitcast(fx.BFloat16) + + PV16_AHEAD = 2 + + def pv16_steps(ph, accs, out): + st_ = {} + NS = len(PV16_SEQ) + + def step(n): + def f(): + blk, rb = PV16_SEQ[n] + if fx.const_expr(n == 0): + sel = rope_w.select( + fx.Int32(ro(ph, NKR, RQ_KR)), + fx.Int32(ro(ph, 2, RQ_V)), + ) + st_["hi"] = [ + [v16_hi[par][qq] + sel for qq in range(2)] + for par in range(2) + ] + st_["acc"] = list(accs) + st_["va"] = {} + st_["pb"] = {} + for b0 in fx.range_constexpr(PV16_AHEAD): + st_["va"][b0] = pv16_a(ph, st_["hi"], b0) + if fx.const_expr(b0 == 0): + for r0 in fx.range_constexpr(RB): + st_["pb"][r0] = pv16_b(ph, r0) + if fx.const_expr(rb == 0 and blk + PV16_AHEAD < 8): + st_["va"][blk + PV16_AHEAD] = pv16_a( + ph, st_["hi"], blk + PV16_AHEAD + ) + ai = blk * RB + rb + st_["acc"][ai] = pv16_mfma( + st_["va"][blk], st_["pb"][rb], st_["acc"][ai] + ) + if fx.const_expr(n == NS - 1): + out.extend(st_["acc"]) + + return f + + return [step(n) for n in range(NS)] + + def al16_read(ph): + ab = ro(ph, 2, ABUF) + return [ + fx.Float32( + lds_rd( + at( + a16_b, + ( + ab + 64 * rb + if isinstance(ab, int) + else ab + fx.Int32(64 * rb) + ), + ), + fx.Vector.make_type(1, fx.Float32), + 4, + )[0] + ) + for rb in range(RB) + ] + + def rescale16(accs, al): + ne = al[0] != fx.Float32(1.0) + for rb in fx.range_constexpr(1, RB): + ne = ne | (al[rb] != fx.Float32(1.0)) + need = fx.Int64( + fx.rocdl.ballot(T.i64, ne.ir_value()) + ) != fx.Int64(0) + if need: + accs = [ + rescale_blk(accs[ai], al[ai % RB], 4) + for ai in range(8 * RB) + ] + return accs + + def ptile(q): + return (q < t1 - t0).select(t0 + q - fx.Int32(1), t1) + + def idx_rd(ph): + return [ + fx.Int32( + lds_rd( + at(ixr_b, ro(ph, NIX, 2048, 256 * u)), + fx.Vector.make_type(1, fx.Int32), + 4, + )[0] + ) + for u in range(NKD) + ] + + def dma_ix(q, ph): + for u, kk in enumerate(kd_list): + ga = ibase + fx.Int64(ix_clamp(ptile(q), kk)) * fx.Int64(4) + _rocdl_d.global_load_lds( + fx.to_llvm_ptr(cu._to_ptr_global(ga)), + fx.to_llvm_ptr( + p_rq + + at( + wv * fx.Int32(512), + ro(ph, NIX, 2048, RQ_IX_OFF + 256 * u), + ) + ), + 4, + 0, + alias_scopes=DMA_AS, + ) + + def dma_kv(rows, ph): + gk = kf_lane + row_u(rows[0]) * fx.Int64(ROWB) + for cg in fx.range_constexpr(4): + dma16( + gk + fx.Int64(128 * cg), + at(wv * fx.Int32(1024), ro(ph, NKF, RQ_KF, 4096 * cg)), + KV_NT, + ) + gr = kr_lane + row_u(rows[1]) * fx.Int64(2 * ROPE) + dma16( + gr, + at(wv * fx.Int32(1024), ro(ph, NKR, RQ_KR, RQ_KR_OFF)), + KV_NT, + ) + + def pv_step(ph, s, accs, live=None, v16=False): + i3 = idx_rd(padd(ph, 3)) + al = al16_read(ph) if v16 else al_read(ph) + fence() + accs = rescale16(accs, al) if v16 else rescale_al(accs, al) + fence() + + def dma_a(): + dma_ix(s + fx.Int32(5), padd(ph, 5)) + + def dma_b(): + dma_kv(i3, padd(ph, 3)) + + side = [dma_a, dma_b] + if fx.const_expr(live is not None): + if live: + dma_b() + fence() + side = [dma_a] + po = [] + run_mixed( + pv16_steps(ph, accs, po) if v16 else pv_steps(ph, accs, po), + side + dequant_steps(padd(ph, 1)), + every=1, + ) + if fx.const_expr(live is None): + fx.rocdl.s_waitcnt(vmcnt=NVM * (DD - 2)) + else: + if live: + fx.rocdl.s_waitcnt(vmcnt=NVM * (DD - 2)) + else: + fx.rocdl.s_waitcnt(vmcnt=NKD * (DD - 2)) + fence() + lds_barrier() + fence() + return po + + nits = t1 - t0 + tp0 = (t1 > t0).select(t1 - fx.Int32(1), t0) + lim0 = (t1 > t0).select(kvlen - tp0 * fx.Int32(KTL), zero_i) + if fx.const_expr(MERGED): + slot, myx, tk_lane = protocol_entry() + acc0z = fx.Vector.filled(16, 0.0, fx.Float32) + + def qk_role(): + q16 = load_q16() + if fx.const_expr(Q16_OSC): + snk_q = fx.Float32( + fx.ptr_load( + sink_ptr + + fx.Int64( + (jb0 * fx.Int32(16) + sm_row) & fx.Int32(H - 1) + ) + ) + ) * fx.Float32(LOG2E) + bnd = None + if fx.const_expr(BOUNDS): + b_qp = _div_const(jb0 + wq, HB) + b_row = q0 + (b_qp < qlen).select(b_qp, qlen - fx.Int32(1)) + bnd = [ + _sload(bnd_ptr, b_row * fx.Int32(4) + fx.Int32(i)) + for i in range(4) + ] + lds_barrier() + m_c, l_c = qk_step( + q16, 0, ninf, fx.Float32(0.0), lim0, tp0, bnd + ) + nq = (nits > fx.Int32(1)).select(nits - fx.Int32(1), zero_i) + ngq = nq >> fx.Int32(2) + for itq, stq in range( + zero_i, + ngq, + fx.Int32(1), + init=[m_c.ir_value(), l_c.ir_value()], + ): + m_q = fx.Float32(stq[0]) + l_q = fx.Float32(stq[1]) + tq = t0 + fx.Int32(itq) * fx.Int32(4) + for u in fx.range_constexpr(4): + m_q, l_q = qk_step( + q16, u + 1, m_q, l_q, None, tq + fx.Int32(u), bnd + ) + rq = yield [m_q.ir_value(), l_q.ir_value()] + m_l = fx.Float32(rq[0]) + l_l = fx.Float32(rq[1]) + remq = nq & fx.Int32(3) + tr0 = t0 + ngq * fx.Int32(4) + for u in fx.range_constexpr(3): + if remq > fx.Int32(u): + m_r, l_r = qk_step( + q16, u + 1, m_l, l_l, None, tr0 + fx.Int32(u), bnd + ) + m_l = m_r + l_l = l_r + if fx.const_expr(Q16_OSC): + lf = l_l.maximumf(fx.Float32(1e-30)) + lse2 = m_l + fly_math.log2(lf) + gm = snk_q.maximumf(lse2) + w_o = exp2(lse2 - gm) + den = exp2(snk_q - gm) + w_o + if fx.const_expr(BOUNDS): + nokey = l_l == fx.Float32(0.0) + else: + nokey = kvlen == zero_i + osc = nokey.select( + qnan, ((fx.Float32(1.0) / lf) * w_o) * rcp(den) + ) + if sm_first: + if fx.const_expr(Q16_OSC): + _llvm_d.StoreOp( + _arith.unwrap(osc), + fx.to_llvm_ptr( + p_rq + + ( + fx.Int32(RD_ML_OFF + 8 * ROWS) + + sm_row * fx.Int32(4) + ) + ), + alignment=4, + ) + _llvm_d.StoreOp( + _arith.unwrap(m_l), + fx.to_llvm_ptr( + p_rq + (fx.Int32(RD_ML_OFF) + sm_row * fx.Int32(4)) + ), + alignment=4, + ) + _llvm_d.StoreOp( + _arith.unwrap(l_l), + fx.to_llvm_ptr( + p_rq + + ( + fx.Int32(RD_ML_OFF + 4 * ROWS) + + sm_row * fx.Int32(4) + ) + ), + alignment=4, + ) + lds_barrier() + lds_barrier() + if fx.const_expr(cfg.PFN): + prefetch_next() + return ( + m_l, + l_l, + [ + fx.Vector( + _llvm_d.InlineAsmOp( + v16f32, [acc0z.ir_value()], "; acc $0", "=a,0" + ).res + ) + for _ in range(0 if PV16R else NACC) + ], + ) + + def pv_role(): + if fx.const_expr(PV16R): + pv_kvall = _sload(indptr_ptr, nseq) - _sload( + indptr_ptr, zero_i + ) + ptiles = [tp0, ptile(fx.Int32(1)), ptile(fx.Int32(2))] + idx0 = [ + [ld_idx(ix_clamp(tt_, kk)) for kk in kd_list] + for tt_ in ptiles + ] + for j in fx.range_constexpr(DD): + dma_ix(fx.Int32(j + 2), j + 2) + dma_kv(idx0[j], j) + fx.rocdl.s_waitcnt(vmcnt=NVM * (DD - 1)) + lds_barrier() + run_gen(dequant_steps(0)) + fx.rocdl.s_waitcnt(vmcnt=NVM * (DD - 2)) + lds_barrier() + if fx.const_expr(PV16R): + lo_, hi_ = PV16R + use16 = _ult(kvlen - fx.Int32(lo_ + 1), fx.Int32(hi_ - lo_)) + use16 = use16 & ( + fx.Int64(kvlen) * fx.Int64(nseq) * fx.Int64(4) + <= fx.Int64(pv_kvall) * fx.Int64(5) + ) + if use16: + q16_store(pv_body(True), True) + else: + q16_store(pv_body(False), False) + return ninf, fx.Float32(0.0), [] + return ninf, fx.Float32(0.0), pv_body(False) + + def pv_body(v16): + nacc = 8 * RB if v16 else NACC + az = fx.Vector.filled(4 if v16 else 16, 0.0, fx.Float32) + acc_c = [ + fx.Vector( + _llvm_d.InlineAsmOp( + v4f32 if v16 else v16f32, + [az.ir_value()], + "; acc $0", + "=a,0", + ).res + ) + for _ in range(nacc) + ] + nva = (nits > zero_i).select(nits, zero_i) + nv = (nva > fx.Int32(DD)).select(nva - fx.Int32(DD), zero_i) + ngv = nv >> fx.Int32(2) + for itv, stv in range( + zero_i, ngv, fx.Int32(1), init=[x.ir_value() for x in acc_c] + ): + acc_v = [fx.Vector(stv[j]) for j in range(nacc)] + sv = fx.Int32(itv) * fx.Int32(4) + for u in fx.range_constexpr(4): + acc_v = pv_step(u, sv + fx.Int32(u), acc_v, v16=v16) + rv = yield [x.ir_value() for x in acc_v] + acc_l = [fx.Vector(rv[j]) for j in range(nacc)] + sv0 = ngv * fx.Int32(4) + remv = nva - sv0 + for u in fx.range_constexpr(DD + 3): + if remv > fx.Int32(u): + acc_r = pv_step( + u, + sv0 + fx.Int32(u), + acc_l, + live=sv0 + fx.Int32(u) < nv, + v16=v16, + ) + acc_l = acc_r + if nits <= zero_i: + lds_barrier() + # drain the PV role's DMAs before the epilogue reuses the tile rings; the s_nops + # cover the inline-asm MFMA -> VALU accumulator read + fx.rocdl.s_waitcnt(vmcnt=0) + lds_barrier() + _llvm_d.InlineAsmOp( + None, + [], + "s_nop 7\n s_nop 7\n s_nop 3", + "", + has_side_effects=True, + ) + return acc_l + + def q16_store(acc_l, v16): + ln_ = fx.Int32( + _llvm_d.InlineAsmOp( + T.i32, + [], + "v_mbcnt_lo_u32_b32 $0, -1, 0\n" + " v_mbcnt_hi_u32_b32 $0, -1, $0", + "=v", + has_side_effects=True, + ).res + ) + m_ = ln_ & fx.Int32(15) + g_ = ln_ >> fx.Int32(4) + hq_ = ln_ >> fx.Int32(5) + r32_ = ln_ & fx.Int32(31) + Q_OBP = 272 + qo_wb = wv * fx.Int32(ROWS * Q_OBP) + + def acc_rd(a, i): + return fx.Float32( + _llvm_d.InlineAsmOp( + T.f32, + [fx.Float32(a[i]).ir_value()], + "v_accvgpr_read_b32 $0, $1", + "=v,a", + has_side_effects=True, + ).res + ) + + for rb in fx.range_constexpr(RB if v16 else NPB): + row = ( + fx.Int32(16 * rb) + m_ + if v16 + else fx.Int32(32 * rb) + r32_ + ) + osc = fx.Float32( + lds_rd( + fx.Int32(RD_ML_OFF + 8 * ROWS) + row * fx.Int32(4), + fx.Vector.make_type(1, fx.Float32), + 4, + )[0] + ) + for nt in fx.range_constexpr(8 if v16 else 16): + fx.rocdl.sched_barrier(0) + if fx.const_expr(v16): + a = acc_l[nt * RB + rb] + av = [acc_rd(a, i) for i in range(4)] + col = fx.Int32(32 * nt) + g_ * fx.Int32(8) + else: + a = acc_l[(nt // 4) * NPB + rb] + av = [acc_rd(a, 4 * (nt % 4) + i) for i in range(4)] + col = fx.Int32( + 2 * (32 * (nt // 4) + 8 * (nt % 4)) + ) + hq_ * fx.Int32(8) + v_ = (fx.Vector.from_elements(av, fx.Float32) * osc).to( + fx.BFloat16 + ) + _llvm_d.StoreOp( + _arith.unwrap(v_.bitcast(fx.Int32)), + fx.to_llvm_ptr( + p_rq + (qo_wb + row * fx.Int32(Q_OBP) + col) + ), + alignment=8, + ) + for k in fx.range_constexpr(ROWS // 4): + qo_rr = fx.Int32(4 * k) + g_ + qo_row = lds_rd( + qo_wb + qo_rr * fx.Int32(Q_OBP) + m_ * fx.Int32(16), + fx.Vector.make_type(4, fx.Int32), + 16, + ) + qo_cr = jb0 * fx.Int32(16) + qo_rr + qo_qp = qo_cr >> fx.Int32(H.bit_length() - 1) + qo_o = fx.Int64(q0 + qo_qp) * fx.Int64(out_s0) + fx.Int64( + qo_cr & fx.Int32(H - 1) + ) * fx.Int64(out_s1) + qo_a = fx.Int64(fx.ptrtoint(out_ptr)) + ( + qo_o + fx.Int64(wv * fx.Int32(128) + m_ * fx.Int32(8)) + ) * fx.Int64(2) + if fx.const_expr(MSQ > 1): + if qo_qp < qlen: + st_agent_vec(qo_a, qo_row, 16) + else: + st_agent_vec(qo_a, qo_row, 16) + + m_l = ninf + l_l = fx.Float32(0.0) + acc_l = [acc0z for _ in range(0 if PV16R else NACC)] + if is_qk: + m_l, l_l, acc_l = qk_role() + else: + m_l, l_l, acc_l = pv_role() + lane_x = fx.Int32( + _llvm_d.InlineAsmOp( + T.i32, + [], + "v_mbcnt_lo_u32_b32 $0, -1, 0\n" + " v_mbcnt_hi_u32_b32 $0, -1, $0", + "=v", + has_side_effects=True, + ).res + ) + tid = wv * fx.Int32(WAVE) + lane_x + wave = tid >> fx.Int32(6) + lane = tid & fx.Int32(WAVE - 1) + m = lane & fx.Int32(15) + g = lane >> fx.Int32(4) + gsh = g * fx.Int32(8) + hq = lane >> fx.Int32(5) + r32 = lane & fx.Int32(31) + wv = fx.Int32(fx.rocdl.readfirstlane(T.i32, wave.ir_value())) + if fx.const_expr(MERGED): + tk_lane = ticket_lane(slot, lane) + else: + idx0 = [idx_tile(t0 + fx.Int32(j)) for j in range(DD)] + if dma_wave: + for j in fx.range_constexpr(DD): + dma_idx(t0 + fx.Int32(DD - 1 + j)) + dma_tile(t0 + fx.Int32(j), idx0[j]) + if fx.const_expr(MERGED): + slot, myx, tk_lane = protocol_entry() + if dma_wave: + fx.rocdl.s_waitcnt(vmcnt=NVM * (DD - 1)) + lds_barrier() + run_gen(dequant_steps(t0)) + if dma_wave: + fx.rocdl.s_waitcnt(vmcnt=NVM * (DD - 2)) + lds_barrier() + sc0l = [] + run_gen(qk_steps(t0, sc0l)) + acc0 = fx.Vector.filled(16, 0.0, fx.Float32) + acc0s = [ + _llvm_d.InlineAsmOp( + v16f32, [acc0.ir_value()], "; acc $0", "=a,0" + ).res + for _ in range(NACC) + ] + NSCV = len(sc0l) + init = [ninf.ir_value(), fx.Float32(0.0).ir_value()] + init += [x.ir_value() for x in sc0l] + init += acc0s + nits = t1 - t0 + nmain = (nits > fx.Int32(1)).select(nits - fx.Int32(1), zero_i) + results = init + AO = 2 + NSCV + for it, state in range(zero_i, nmain, fx.Int32(1), init=init): + m_run = fx.Float32(state[0]) + l_run = fx.Float32(state[1]) + sc_cur = [fx.Vector(state[2 + i]) for i in range(NSCV)] + accs = [fx.Vector(state[AO + j]) for j in range(NACC)] + tt = t0 + fx.Int32(it) + if dma_wave: + i3 = idx_lds(tt + fx.Int32(DD)) + dma_idx(tt + fx.Int32(2 * DD - 1)) + dma_tile(tt + fx.Int32(DD), i3) + fence() + qo = [] + so = [] + m_new = m_run + l_new = l_run + sc_nxt = sc_cur + if qk_wave: + run_mixed( + qk_steps(tt + fx.Int32(1), qo), + softmax_steps(tt, sc_cur, m_run, l_run, False, so), + every=1, + ) + m_new = so[0] + l_new = so[1] + sc_nxt = qo + lds_barrier() + accs = rescale_d(accs) + fence() + po = [] + dqs_ = dequant_steps(tt + fx.Int32(1)) + run_mixed( + pv_steps(tt, accs, po), + dqs_, + every=max(1, (8 * NPB) // len(dqs_)), + ) + if dma_wave: + fx.rocdl.s_waitcnt(vmcnt=NVM * (DD - 2)) + lds_barrier() + results = yield ( + [m_new.ir_value(), l_new.ir_value()] + + [x.ir_value() for x in sc_nxt] + + [x.ir_value() for x in po] + ) + m_l = fx.Float32(results[0]) + l_l = fx.Float32(results[1]) + acc_l = [fx.Vector(results[AO + j]) for j in range(NACC)] + if nits > zero_i: + tl = t1 - fx.Int32(1) + m_l2 = m_l + l_l2 = l_l + if qk_wave: + so = [] + run_gen( + softmax_steps( + tl, + [fx.Vector(results[2 + i]) for i in range(NSCV)], + m_l, + l_l, + True, + so, + ) + ) + m_l2 = so[0] + l_l2 = so[1] + lds_barrier() + acc_l2 = rescale_d(acc_l) + po = [] + run_gen(pv_steps(tl, acc_l2, po, last=True)) + m_l = m_l2 + l_l = l_l2 + acc_l = po + if fx.const_expr(not Q16): + lds_barrier() + if fx.const_expr(Q16): + pass + elif qk_wave & sm_first: + _llvm_d.StoreOp( + _arith.unwrap(m_l), + fx.to_llvm_ptr( + p_rq + (fx.Int32(RD_ML_OFF) + sm_row * fx.Int32(4)) + ), + alignment=4, + ) + _llvm_d.StoreOp( + _arith.unwrap(l_l), + fx.to_llvm_ptr( + p_rq + + (fx.Int32(RD_ML_OFF + 4 * ROWS) + sm_row * fx.Int32(4)) + ), + alignment=4, + ) + if fx.const_expr(not Q16): + lds_barrier() + m_fin, l_fin, acc_fin = [], [], [] + for rb in fx.range_constexpr(0 if PV16R else NPB): + row = fx.Int32(32 * rb) + r32 + m_fin.append( + fx.Float32( + lds_rd( + fx.Int32(RD_ML_OFF) + row * fx.Int32(4), + fx.Vector.make_type(1, fx.Float32), + 4, + )[0] + ) + ) + l_fin.append( + fx.Float32( + lds_rd( + fx.Int32(RD_ML_OFF + 4 * ROWS) + row * fx.Int32(4), + fx.Vector.make_type(1, fx.Float32), + 4, + )[0] + ) + ) + acc_fin.append(_AccRow(acc_l, NPB, rb, Q16)) + elif fx.const_expr(KL): + wl = wave + key_l = wl * fx.Int32(16) + m + + def vphys(kk): + lo = kk & fx.Int32(15) + return ( + (kk - lo) + + ((lo & fx.Int32(3)) << fx.Int32(1)) + + (lo & fx.Int32(8)) + + ((lo >> fx.Int32(2)) & fx.Int32(1)) + ) + + vrow = p_v + vphys(key_l) * fx.Int32(VPITCH) + rrow0 = vphys(g * fx.Int32(8) + m // fx.Int32(4)) + + def idx_row1(t, key): + kidx = t * fx.Int32(KT_K) + key + ok = (t < t1) & (kidx < kvlen) + return ld_idx(ok.select(kidx, zero_i)) + + up8 = m >= fx.Int32(8) + colc = up8.select(g + fx.Int32(4), g) * fx.Int32(16) + + def idx_row(t): + if fx.const_expr(KC): + ka = wl * fx.Int32(16) + (m & fx.Int32(7)) + return [idx_row1(t, ka), idx_row1(t, ka + fx.Int32(8))] + return [idx_row1(t, key_l)] + + def load_kv(rows): + rows = [row_ok(r) for r in rows] + if fx.const_expr(KC): + rowa, rowb = rows + ba = fx.Int64(rowa) * fx.Int64(ROWB) + fx.Int64(colc) + bb = fx.Int64(rowb) * fx.Int64(ROWB) + fx.Int64(colc) + lo, hi = [], [] + for cc in fx.range_constexpr(4): + if fx.const_expr(cc < 3): + oa, ob = ba, bb + else: + c3 = up8.select(fx.Int32(64), g * fx.Int32(16)) - colc + oa, ob = ba + fx.Int64(c3), bb + fx.Int64(c3) + lo.append(load16(kv_ptr, oa + fx.Int64(cc * 128))) + hi.append(load16(kv_ptr, ob + fx.Int64(cc * 128))) + rown = up8.select(rowb, rowa) + rope = [ + fx.ptr_load( + kvr_ptr + + fx.Int64(rown) * fx.Int64(ROPE) + + fx.Int64(g * fx.Int32(8) + fx.Int32(32 * j)), + result_type=v16u8, + ).bitcast(fx.Int32) + for j in range(2) + ] + return lo + hi + rope + kbase = fx.Int64(rows[0]) * fx.Int64(ROWB) + regs = [] + for cc in fx.range_constexpr(4): + regs.append( + load16( + kv_ptr, + kbase + fx.Int64(cc * 128) + fx.Int64(g * fx.Int32(16)), + ) + ) + for cc in fx.range_constexpr(3): + regs.append( + load16( + kv_ptr, + kbase + + fx.Int64(cc * 128 + 64) + + fx.Int64(g * fx.Int32(16)), + ) + ) + regs.append(load16(kv_ptr, kbase + fx.Int64(NOPE))) + krbase = fx.Int64(rows[0]) * fx.Int64(ROPE) + for j in fx.range_constexpr(2): + regs.append( + fx.ptr_load( + kvr_ptr + + krbase + + fx.Int64(g * fx.Int32(8) + fx.Int32(32 * j)), + result_type=v16u8, + ).bitcast(fx.Int32) + ) + return regs + + def kv_fix(raw): + if fx.const_expr(not KC): + return raw + a, b = raw[0:4], raw[4:8] + + def mix(own, oth, bank_mask): + return fx.Vector.from_elements( + [ + fx.Int32( + fx.rocdl.update_dpp( + T.i32, + fx.Int32(own[i]).ir_value(), + fx.Int32(oth[i]).ir_value(), + 0x128, + 0xF, + bank_mask, + False, + ) + ) + for i in range(4) + ], + fx.Int32, + ) + + lo = [mix(a[cc], b[cc], 0xC) for cc in range(4)] + hi = [mix(b[cc], a[cc], 0x3) for cc in range(4)] + return lo + hi + [raw[8], raw[9]] + + NKV = 10 + NR = 2 if KC else 1 + rows_pf = [idx_row(t0 + fx.Int32(j)) for j in range(2)] + + qfr, qscl, qrf = load_q() + fx.rocdl.sched_barrier(0) + kv_pf = load_kv(rows_pf[0]) + + if fx.const_expr(MERGED): + slot, myx, tk_lane = protocol_entry() + + def tile_step(kvc, tt, m_run, l_run, accs): + kvc = kv_fix(kvc) + tvalid = tt < t1 + tbase = tt * fx.Int32(KT_K) + klo = kvc[0:4] + khi = kvc[4:7] + [zero4] + ksc = kvc[7] + krf = [kvc[8].bitcast(fx.BFloat16), kvc[9].bitcast(fx.BFloat16)] + kop = [join8(klo[cc], khi[cc]) for cc in range(4)] + kscl = [_scl(ksc[cc], cc) for cc in range(4)] + scs = [] + for rb in fx.range_constexpr(RB): + sc = fx.Vector.filled(4, 0.0, fx.Float32) + for cc in fx.range_constexpr(4): + sc = fx.Vector( + fx.rocdl.mfma_scale_f32_16x16x128_f8f6f4( + v4f32, + [ + kop[cc], + qfr[rb][cc], + sc, + 0, + 0, + 0, + kscl[cc], + 0, + qscl[rb][cc], + ], + ) + ) + for j in fx.range_constexpr(2): + sc = fx.Vector( + fx.rocdl.mfma_f32_16x16x32_bf16( + v4f32, [krf[j], qrf[rb][j], sc] + ) + ) + scs.append(sc) + for cc in fx.range_constexpr(4): + for hh in fx.range_constexpr(2 if cc < 3 else 1): + blk = 2 * cc + hh + e = ( + fx.Int32(ksc[blk // 2]) >> fx.Int32(16 * (blk % 2)) + ) & (fx.Int32(0xFF)) + scl = (e << fx.Int32(23)).bitcast(fx.Float32) + src = klo[cc] if hh == 0 else khi[cc] + parts = [] + for wd in fx.range_constexpr(4): + for sel in fx.range_constexpr(2): + pr = fx.Vector( + fx.rocdl.cvt_scalef32_pk_bf16_fp8( + v2bf16, + fx.Int32(src[wd]).ir_value(), + scl.ir_value(), + bool(sel), + ) + ) + parts += [pr[0], pr[1]] + vv = fx.Vector.from_elements(parts, fx.BFloat16) + fx.ptr_store( + vv, + vrow + fx.Int32(cc * 128 + hh * 64) + g * fx.Int32(16), + ) + for j in fx.range_constexpr(2): + fx.ptr_store( + krf[j], vrow + fx.Int32(NOPE + 32 * j) + g * fx.Int32(8) + ) + flat = [ + fx.Float32(scs[rb][i]) * scale_log2e + for rb in range(RB) + for i in range(4) + ] + tend = tbase + fx.Int32(KT_K) + kfull = (tend <= kvlen) & tvalid + if fx.const_expr(BOUNDS): + for rb in fx.range_constexpr(RB): + a0_, a1_, b0_, b1_ = rb_bnd[rb] + kfull = kfull & ( + ((a0_ <= tbase) & (a1_ >= tend)) + | ((b0_ <= tbase) & (b1_ >= tend)) + ) + kpart = fx.Int64( + fx.rocdl.ballot(T.i64, (kfull == false_).ir_value()) + ) != fx.Int64(0) + if kpart: + kval = [] + for i in fx.range_constexpr(4): + kk = ( + tbase + + wl * fx.Int32(16) + + g * fx.Int32(4) + + fx.Int32(i) + ) + kval.append((kk < kvlen) & tvalid) + mflat = [] + for rb in fx.range_constexpr(RB): + kv_rb = kval + if fx.const_expr(BOUNDS): + kv_rb = [ + kval[i] + & in_bounds( + tbase + + wl * fx.Int32(16) + + g * fx.Int32(4) + + fx.Int32(i), + rb_bnd[rb], + ) + for i in range(4) + ] + mflat += [ + kv_rb[i].select(flat[4 * rb + i], ninf) + for i in range(4) + ] + flat = mflat + ss = [] + for rb in fx.range_constexpr(RB): + s = flat[4 * rb : 4 * rb + 4] + ss.append(s) + mx = s[0].maximumf(s[1]).maximumf(s[2]).maximumf(s[3]) + mx = plswap_f(mx, 16, _fmax) + mx = plswap_f(mx, 32, _fmax) + if g == zero_i: + lds_st( + p_rmax, wl * fx.Int32(ROWS) + fx.Int32(16 * rb) + m, mx + ) + fx.gpu.barrier() + m_new, alpha = [], [] + lsum = [] + for rb in fx.range_constexpr(RB): + tmax = lds_ldf(p_rmax, fx.Int32(16 * rb) + m) + for ww in fx.range_constexpr(1, NWAVES): + tmax = tmax.maximumf( + lds_ldf(p_rmax, fx.Int32(ww * ROWS + 16 * rb) + m) + ) + mn = (tmax > m_run[rb] + fx.Float32(8.0)).select( + tmax, m_run[rb] + ) + m_safe = (mn == ninf).select(fx.Float32(0.0), mn) + m_new.append(mn) + alpha.append(exp2(m_run[rb] - m_safe)) + p = [exp2(ss[rb][i] - m_safe) for i in range(4)] + pb = fx.Vector.from_elements(p, fx.Float32).to(fx.BFloat16) + fx.ptr_store( + pb, + p_p + + (fx.Int32(16 * rb) + m) * fx.Int32(PPITCH) + + wl * fx.Int32(16) + + g * fx.Int32(4), + ) + pr = pb.to(fx.Float32) + ps = ( + fx.Float32(pr[0]) + + fx.Float32(pr[1]) + + fx.Float32(pr[2]) + + fx.Float32(pr[3]) + ) + lsum.append(ps) + fx.gpu.barrier() + l_new, pfr = [], [] + for rb in fx.range_constexpr(RB): + l_new.append(l_run[rb] * alpha[rb] + lsum[rb]) + pfr.append( + [ + fx.ptr_load( + p_p + + (fx.Int32(16 * rb) + m) * fx.Int32(PPITCH) + + fx.Int32(32 * ks) + + g * fx.Int32(8), + result_type=v8bf16, + ) + for ks in range(2) + ] + ) + ne = alpha[0] != fx.Float32(1.0) + for rb in fx.range_constexpr(1, RB): + ne = ne | (alpha[rb] != fx.Float32(1.0)) + need = fx.Int64(fx.rocdl.ballot(T.i64, ne.ir_value())) != fx.Int64( + 0 + ) + accs = [ + [fx.Vector(accs[rb][nt]) for nt in range(8)] for rb in range(RB) + ] + if need: + accs = [ + [accs[rb][nt] * alpha[rb] for nt in range(8)] + for rb in range(RB) + ] + new_accs = [[None] * 8 for _ in range(RB)] + for nt in fx.range_constexpr(8): + a = [fx.Vector(accs[rb][nt]) for rb in range(RB)] + dvb = wl * fx.Int32(128) + fx.Int32(16 * nt) + for ks in fx.range_constexpr(2): + halves = [] + for hh in fx.range_constexpr(2): + rk = rrow0 + fx.Int32(32 * ks + hh) + ptr = ( + p_v + + rk * fx.Int32(VPITCH) + + dvb + + (m % fx.Int32(4)) * fx.Int32(4) + ) + halves.append( + fx.Vector( + fx.rocdl.ds_read_tr16_b64( + v4bf16, fx.to_llvm_ptr(ptr) + ).result + ) + ) + va = fx.Vector.from_elements( + [halves[0][i] for i in range(4)] + + [halves[1][i] for i in range(4)], + fx.BFloat16, + ) + for rb in fx.range_constexpr(RB): + a[rb] = fx.Vector( + fx.rocdl.mfma_f32_16x16x32_bf16( + v4f32, [va, pfr[rb][ks], a[rb]] + ) + ) + for rb in fx.range_constexpr(RB): + new_accs[rb][nt] = a[rb] + fx.gpu.barrier() + return m_new, l_new, new_accs + + NST = 2 * RB + 8 * RB + init = [ninf.ir_value() for _ in range(RB)] + init += [fx.Float32(0.0).ir_value() for _ in range(RB)] + init += [ + fx.Vector.filled(4, 0.0, fx.Float32).ir_value() + for _ in range(8 * RB) + ] + init += [x.ir_value() for x in kv_pf] + [ + x.ir_value() for x in rows_pf[1] + ] + nits = t1 - t0 + results = init + for ip, state in range( + zero_i, nits >> fx.Int32(1), fx.Int32(1), init=init + ): + m_run = [fx.Float32(state[rb]) for rb in range(RB)] + l_run = [fx.Float32(state[RB + rb]) for rb in range(RB)] + accs = [ + [fx.Vector(state[2 * RB + 8 * rb + nt]) for nt in range(8)] + for rb in range(RB) + ] + kvq = [[fx.Vector(state[NST + i]) for i in range(NKV)]] + rows_n = [fx.Int32(state[NST + NKV + i]) for i in range(NR)] + tt = t0 + (fx.Int32(ip) << fx.Int32(1)) + for half in fx.range_constexpr(2): + rows_nn = idx_row(tt + fx.Int32(2 + half)) + kvq.append(load_kv(rows_n)) + m_run, l_run, accs = tile_step( + kvq[half], tt + fx.Int32(half), m_run, l_run, accs + ) + rows_n = rows_nn + results = yield ( + [x.ir_value() for x in m_run] + + [x.ir_value() for x in l_run] + + [x.ir_value() for rb in range(RB) for x in accs[rb]] + + [x.ir_value() for x in kvq[2]] + + [x.ir_value() for x in rows_n] + ) + m_fin = [fx.Float32(results[rb]) for rb in range(RB)] + l_fin = [fx.Float32(results[RB + rb]) for rb in range(RB)] + acc_fin = [ + [fx.Vector(results[2 * RB + 8 * rb + nt]) for nt in range(8)] + for rb in range(RB) + ] + kv_last = [fx.Vector(results[NST + i]) for i in range(NKV)] + if (nits & fx.Int32(1)) != zero_i: + m_fin, l_fin, acc_fin = tile_step( + kv_last, t1 - fx.Int32(1), m_fin, l_fin, acc_fin + ) + for rb in fx.range_constexpr(RB): + ls = plswap_f(l_fin[rb], 16, _fadd) + ls = plswap_f(ls, 32, _fadd) + if g == zero_i: + lds_st(p_rsum, wl * fx.Int32(ROWS) + fx.Int32(16 * rb) + m, ls) + fx.gpu.barrier() + l_tot = [] + for rb in fx.range_constexpr(RB): + tsum = lds_ldf(p_rsum, fx.Int32(16 * rb) + m) + for ww in fx.range_constexpr(1, NWAVES): + tsum = tsum + lds_ldf(p_rsum, fx.Int32(ww * ROWS + 16 * rb) + m) + l_tot.append(tsum) + l_fin = l_tot + + empty = kvlen == zero_i + + NACCW = 64 if RQ else (16 if RD else 8) + NRBE = 1 if RQ else (ROWS // 32 if RD else RB) + em = r32 if (RQ or RD) else m + eg = hq if (RQ or RD) else g + RPB = 16 if (RQ or RD) else 8 + ROPE_RUNS = 8 if (RQ or RD) else 4 + + def rope_off(k): + if fx.const_expr(RQ or RD): + return fx.Int32(32 * (k // 4) + 8 * (k % 4)) + hq * fx.Int32(4) + return fx.Int32(16 * k) + g * fx.Int32(4) + + def dv_of(nt): + if fx.const_expr(RQ): + return fx.Int32(32 * (nt // 4) + 8 * (nt % 4)) + hq * fx.Int32(4) + if fx.const_expr(RD): + return ( + wv * fx.Int32(128) + + fx.Int32(32 * (nt // 4) + 8 * (nt % 4)) + + hq * fx.Int32(4) + ) + return wl * fx.Int32(128) + fx.Int32(16 * nt) + g * fx.Int32(4) + + def direct_epi(): + if fx.const_expr(not Q16): + snk = [ + fx.Float32(fx.ptr_load(sink_ptr + fx.Int64(rb_h0[rb] + em))) + * fx.Float32(LOG2E) + for rb in range(NRBE) + ] + if fx.const_expr(EPI_Q): + frv = [ + [ + fx.ptr_load( + fr_ptr + + rb_pos[rb] * fx.Int64(ROPE) + + fx.Int64(rope_off(j)), + result_type=v4f32, + ) + for j in range(ROPE_RUNS) + ] + for rb in range(NRBE if QROW_EPI else 1) + ] + if fx.const_expr(RQ): + is_rw = true_ + elif fx.const_expr(RD): + is_rw = wv == fx.Int32(3) + else: + is_rw = wl == fx.Int32(3) + for rb in fx.range_constexpr(NRBE): + if fx.const_expr(Q16): + osc = fx.Float32( + lds_rd( + fx.Int32(RD_ML_OFF + 8 * ROWS) + + (fx.Int32(32 * rb) + r32) * fx.Int32(4), + fx.Vector.make_type(1, fx.Float32), + 4, + )[0] + ) + inv_l, empty_r = osc, empty + else: + lf = l_fin[rb].maximumf(fx.Float32(1e-30)) + lse2 = m_fin[rb] + fly_math.log2(lf) + gm = snk[rb].maximumf(lse2) + w = exp2(lse2 - gm) + den = exp2(snk[rb] - gm) + w + inv_l = fx.Float32(1.0) / lf + empty_r = (l_fin[rb] == fx.Float32(0.0)) if BOUNDS else empty + f = empty_r.select(qnan, rcp(den)) + h = rb_h0[rb] + em + if fx.const_expr(Q16 and not EPI_Q): + Q_OBP = 272 + qo_wb = wv * fx.Int32(ROWS * Q_OBP) + if wv < fx.Int32(4): + for nt in fx.range_constexpr(NACCW): + if fx.const_expr(nt % 2 == 0 or MERGED): + fx.rocdl.sched_barrier(0) + v_ = (acc_fin[rb][nt] * osc).to(fx.BFloat16) + _llvm_d.StoreOp( + _arith.unwrap(v_.bitcast(fx.Int32)), + fx.to_llvm_ptr( + p_rq + + ( + qo_wb + + (fx.Int32(32 * rb) + r32) + * fx.Int32(Q_OBP) + + fx.Int32( + 2 * (32 * (nt // 4) + 8 * (nt % 4)) + ) + + hq * fx.Int32(8) + ) + ), + alignment=8, + ) + # all LDS reads ahead of the (asm) stores: one wait + qo_rows = [ + lds_rd( + qo_wb + + (fx.Int32(32 * rb + 4 * k) + g) * fx.Int32(Q_OBP) + + m * fx.Int32(16), + fx.Vector.make_type(4, fx.Int32), + 16, + ) + for k in range(8) + ] + for k in fx.range_constexpr(8): + qo_rr = fx.Int32(32 * rb + 4 * k) + g + qo_row = qo_rows[k] + qo_cr = jb0 * fx.Int32(16) + qo_rr + qo_qp = qo_cr >> fx.Int32(H.bit_length() - 1) + qo_o = fx.Int64(q0 + qo_qp) * fx.Int64( + out_s0 + ) + fx.Int64(qo_cr & fx.Int32(H - 1)) * fx.Int64(out_s1) + qo_a = fx.Int64(fx.ptrtoint(out_ptr)) + ( + qo_o + + fx.Int64(wv * fx.Int32(128) + m * fx.Int32(8)) + ) * fx.Int64(2) + if fx.const_expr(MSQ > 1): + if qo_qp < qlen: + st_agent_vec(qo_a, qo_row, 16) + else: + st_agent_vec(qo_a, qo_row, 16) + elif fx.const_expr(not EPI_Q): + obase = fx.Int64(rb_qrow[rb]) * fx.Int64(out_s0) + fx.Int64( + h + ) * fx.Int64(out_s1) + if rb_ok[rb]: + for nt in fx.range_constexpr(NACCW): + fx.ptr_store( + ((acc_fin[rb][nt] * inv_l) * w * f).to(fx.BFloat16), + out_ptr + obase + fx.Int64(dv_of(nt)), + ) + else: + qrow_x = rb_qrow[rb] if QROW_EPI else seq + fr_rb = frv[rb if QROW_EPI else 0] + # Q16: the PV waves (0-3) hold the rows; QROW_EPI: rows past qlen + st_ok = rb_ok[rb] if QROW_EPI else true_ + if fx.const_expr(Q16): + st_ok = st_ok & (wv < fx.Int32(4)) + xrow = fx.Int64(qrow_x) * fx.Int64(H) + fx.Int64(h) + for blk in fx.range_constexpr(NACCW // RPB): + ys = [] + amb = zero_i + for n8 in fx.range_constexpr(RPB): + nt = RPB * blk + n8 + x = acc_fin[rb][nt] * inv_l + if fx.const_expr(not Q16): + x = (x * w) * f + x = x.to(fx.BFloat16).to(fx.Float32) + if fx.const_expr(nt >= NACCW - ROPE_RUNS): + fr = fr_rb[nt - (NACCW - ROPE_RUNS)] + yy = [] + for pp in fx.range_constexpr(2): + c = fx.Float32(fr[2 * pp]) + sn = fx.Float32(fr[2 * pp + 1]) + xa = fx.Float32(x[2 * pp]) + xb = fx.Float32(x[2 * pp + 1]) + yy.append(is_rw.select(xa * c + xb * sn, xa)) + yy.append(is_rw.select(xb * c - xa * sn, xb)) + x = fx.Vector.from_elements(yy, fx.Float32) + y = x.to(fx.BFloat16).to(fx.Float32) + ys.append(y) + for i in fx.range_constexpr(4): + amb = smax_i32(amb, fabs_bits(fx.Float32(y[i]))) + if fx.const_expr(not RQ): + amb = plswap(amb, 16, smax_i32) + amb = plswap(amb, 32, smax_i32) + ex = mxfp8_exp(amb) + ex = empty_r.select(fx.Int32(255), ex) + nexp = fx.Int32(127) - ex + xq_base = xrow * fx.Int64(DV) + for n8 in fx.range_constexpr(RPB): + nt = RPB * blk + n8 + y = ys[n8] + qv = [ + fp8_scaled(fx.Float32(y[i]), nexp) for i in range(4) + ] + pk = fx.rocdl.cvt_pk_fp8_f32( + T.i32, qv[0], qv[1], fx.Int32(0), False + ) + pk = fx.rocdl.cvt_pk_fp8_f32( + T.i32, qv[2], qv[3], pk, True + ) + pk = empty_r.select(fx.Int32(0x7F7F7F7F), fx.Int32(pk)) + pkb = fx.Vector.from_elements([pk], fx.Int32).bitcast( + fx.Uint8 + ) + if fx.const_expr(QROW_EPI or Q16): + if st_ok: + fx.ptr_store( + pkb, out_ptr + xq_base + fx.Int64(dv_of(nt)) + ) + else: + fx.ptr_store( + pkb, out_ptr + xq_base + fx.Int64(dv_of(nt)) + ) + xs_a = ( + xs_ptr + + xrow * fx.Int64(4) + + ( + fx.Int64(blk) + if RQ + else (fx.Int64(wv) if RD else fx.Int64(wl)) + ) + ) + xs_ok = eg == zero_i + if fx.const_expr(QROW_EPI or Q16): + xs_ok = xs_ok & st_ok + if xs_ok: + fx.ptr_store(fx.Uint8(ex), xs_a) + + def pst(addr, val, nbytes): + if fx.const_expr(WTP): + st_agent_vec(addr, val, nbytes) + elif fx.const_expr(SK): + wthru = sk_wt + if wthru: + st_agent_vec(addr, val, nbytes) + else: + st_raw(addr, val, nbytes) + else: + st_raw(addr, val, nbytes) + + def pld(addr, n): + if fx.const_expr(WTP): + return ld_agent_f32(addr, n) + return gload(addr, n) + + def pld16(addr): + if fx.const_expr(WTP): + v = (ld64_agent if EPT == 4 else ld_agent_i32)(addr, False) + return fx.Vector.from_elements([v], type(v)).bitcast(fx.Float16) + return fx.generic_load( + cu._to_ptr_global(addr, fx.Float16, 2 * EPT), count=EPT + ) + + def merged_epi(): + tkv = zero_i + if fx.const_expr(not OWN): # noqa: SIM102 (compile-time guard) + if wave == zero_i: + tkv = ld_agent_i32(tk_lane, volatile=False) + sub = tid // fx.Int32(TPB) + lb = tid % fx.Int32(TPB) + + bpcv = ( + udiv(fx.Int32(NQB) + ns - fx.Int32(1), ns) if DYN else fx.Int32(bpc) + ) + + def slice_addrs(j, pin): + qa = j * bpcv + qe = (qa + bpcv < fx.Int32(NQB)).select(qa + bpcv, fx.Int32(NQB)) + rs = [] + for r in fx.range_constexpr(NRND): + qb = qa + fx.Int32(BPR * r) + sub + act = qb < qe + qbs = act.select(qb, zero_i) + row = qbs >> fx.Int32(2) + cblk = qbs & fx.Int32(3) + jb = jb0 + (row >> fx.Int32(4)) + qpos = _div_const(jb, HB) + head = _rem_const(jb, HB) * fx.Int32(16) + (row & fx.Int32(15)) + d0 = cblk * fx.Int32(128) + lb * fx.Int32(EPT) + la = lse_base + ( + fx.Int64(group) * fx.Int64(ROWS) + fx.Int64(row) + ) * fx.Int64(SP * 4) + pbase = part_base + ( + (fx.Int64(group) * fx.Int64(S * ROWS) + fx.Int64(row)) + * fx.Int64(DV) + + fx.Int64(d0) + ) * fx.Int64(PEB) + pa = [ + pbase + fx.Int64(sidx * ROWS * DV * PEB) + for sidx in range(S) + ] + if fx.const_expr(pin): + la = pin_v64(la) + pa = [pin_v64(x) for x in pa] + sa = fx.Int64(fx.ptrtoint(sink_ptr)) + fx.Int64( + head + ) * fx.Int64(4) + frr = None + if fx.const_expr(EPI_Q): + ro = (d0 >= fx.Int32(NOPE)).select( + d0 - fx.Int32(NOPE), zero_i + ) + if fx.const_expr(QROW_EPI): + qpc = (qpos < qlen).select(qpos, qlen - fx.Int32(1)) + pos_r = fx.Int64( + fx.ptr_load(pos_ptr + fx.Int64(q0 + qpc)) + ) + else: + pos_r = pos + frr = fx.ptr_load( + fr_ptr + pos_r * fx.Int64(ROPE) + fx.Int64(ro), + result_type=fx.Vector.make_type(EPT, fx.Float32), + ) + psa = None + if fx.const_expr(PF16): + psa = psc_base + ( + (fx.Int64(group) * fx.Int64(ROWS) + fx.Int64(row)) + * fx.Int64(4) + + fx.Int64(cblk) + ) * fx.Int64(SP * 4) + rs.append( + _SliceRow(act, head, qpos, d0, la, pa, sa, frr, cblk, psa) + ) + return rs + + def merge_slice(rs): + for c0 in MCHUNKS: + if fx.const_expr(DYN and c0 > 0): + if fx.Int32(c0 * BPR) < bpcv: + merge_rounds(rs, list(range(c0, min(NRND, c0 + MCH)))) + else: + merge_rounds(rs, list(range(c0, min(NRND, c0 + MCH)))) + + def merge_finish(r, rs, acc, den, lv): + x = rs[r] + act, head, qpos, d0 = x.act, x.head, x.qpos, x.d0 + frr, cblk = x.frr, x.cblk + if fx.const_expr(BOUNDS): + gl = lv[0] + for sidx in fx.range_constexpr(1, S): + gl = gl.maximumf(lv[sidx]) + empty_r = gl == ninf + else: + empty_r = empty + fsc = empty_r.select(qnan, rcp(den)) + if fx.const_expr(not EPI_Q): + if act & (qpos < qlen): + fx.ptr_store( + (acc * fsc).to(fx.BFloat16), + out_ptr + + fx.Int64(q0 + qpos) * fx.Int64(out_s0) + + fx.Int64(head) * fx.Int64(out_s1) + + fx.Int64(d0), + ) + else: + x = (acc * fsc).to(fx.BFloat16).to(fx.Float32) + is_rope = d0 >= fx.Int32(NOPE) + ys = [] + for pp in fx.range_constexpr(EPT // 2): + c = fx.Float32(frr[2 * pp]) + sn = fx.Float32(frr[2 * pp + 1]) + xa = fx.Float32(x[2 * pp]) + xb = fx.Float32(x[2 * pp + 1]) + ys.append(is_rope.select(xa * c + xb * sn, xa)) + ys.append(is_rope.select(xb * c - xa * sn, xb)) + y = ( + fx.Vector.from_elements(ys, fx.Float32) + .to(fx.BFloat16) + .to(fx.Float32) + ) + amb = fabs_bits(fx.Float32(y[0])) + for i in fx.range_constexpr(1, EPT): + amb = smax_i32(amb, fabs_bits(fx.Float32(y[i]))) + amb = butterfly_i32(amb, TPB, smax_i32) + ex = empty_r.select(fx.Int32(255), mxfp8_exp(amb)) + nexp = fx.Int32(127) - ex + qv = [fp8_scaled(fx.Float32(y[i]), nexp) for i in range(EPT)] + pk = fx.rocdl.cvt_pk_fp8_f32( + T.i32, qv[0], qv[1], fx.Int32(0), False + ) + if fx.const_expr(EPT == 4): + pk = fx.rocdl.cvt_pk_fp8_f32(T.i32, qv[2], qv[3], pk, True) + pk = empty_r.select(fx.Int32(0x7F7F7F7F), fx.Int32(pk)) + xqrow = (q0 + qpos) if QROW_EPI else seq + act_x = (act & (qpos < qlen)) if QROW_EPI else act + if act_x: + xq_off = ( + fx.Int64(xqrow) * fx.Int64(H) + fx.Int64(head) + ) * fx.Int64(DV) + fx.Int64(d0) + pkb = fx.Vector.from_elements([pk], fx.Int32).bitcast( + fx.Uint8 + ) + if fx.const_expr(EPT == 4): + fx.ptr_store(pkb, out_ptr + xq_off) + else: + fx.ptr_store( + fx.Vector.from_elements([pkb[0], pkb[1]], fx.Uint8), + out_ptr + xq_off, + ) + if lb == zero_i: + fx.ptr_store( + fx.Uint8(ex), + xs_ptr + + (fx.Int64(xqrow) * fx.Int64(H) + fx.Int64(head)) + * fx.Int64(4) + + fx.Int64(cblk), + ) + + def merge_rounds_sk(rs, rounds): + + def pass_loads(r, sb, n): + la, pa0 = rs[r].la, rs[r].pa[0] + lsv = [] + for k4 in fx.range_constexpr(n // 4): + v4 = fx.Vector( + pld( + la + fx.Int64(sb + fx.Int32(4 * k4)) * fx.Int64(4), + 4, + ) + ) + lsv += [fx.Float32(v4[i]) for i in range(4)] + lse, pv = [], [] + for i in fx.range_constexpr(n): + si = sb + fx.Int32(i) + ok = (tps * si) < ntiles + lse.append(ok.select(lsv[i], ninf)) + pa = pa0 + fx.Int64(ok.select(si, zero_i)) * fx.Int64( + ROWS * DV * PEB + ) + pv.append(fx.Vector(pld(pa, EPT))) + return lse, pv + + def pass_add(st, lse, pv): + m, den, acc = st + mn = m + for x in lse: + mn = mn.maximumf(x) + al = exp2(m - mn) + den = den * al + acc = acc * al + for x, p in zip(lse, pv): + w = exp2(x - mn) + den = den + w + acc = acc + p * w + return mn, den, acc + + first = {} + for r in rounds: + snk = fx.Float32(gload(rs[r].sa, 1)) + first[r] = (snk, pass_loads(r, zero_i, SKP0)) + fx.rocdl.sched_barrier(0) + zv = fx.Vector.filled(EPT, 0.0, fx.Float32) + fin = {} + for r in rounds: + snk, (lse, pv) = first[r] + snk = snk * fx.Float32(LOG2E) + fin[r] = pass_add((snk, fx.Float32(1.0), zv), lse, pv) + if fx.const_expr(S > SKP0): + init = [] + for r in rounds: + init += [x.ir_value() for x in fin[r]] + npass = (ns + fx.Int32(3 - SKP0)) >> fx.Int32(2) + results = init + for pp, state in range(zero_i, npass, fx.Int32(1), init=init): + sb = fx.Int32(pp) * fx.Int32(4) + fx.Int32(SKP0) + ld = {r: pass_loads(r, sb, 4) for r in rounds} + fx.rocdl.sched_barrier(0) + nxt = [] + for ir, r in enumerate(rounds): + st = ( + fx.Float32(state[3 * ir]), + fx.Float32(state[3 * ir + 1]), + fx.Vector(state[3 * ir + 2]), + ) + nxt += [x.ir_value() for x in pass_add(st, *ld[r])] + results = yield nxt + for ir, r in enumerate(rounds): + fin[r] = ( + fx.Float32(results[3 * ir]), + fx.Float32(results[3 * ir + 1]), + fx.Vector(results[3 * ir + 2]), + ) + for r in rounds: + _, den, acc = fin[r] + merge_finish(r, rs, acc, den, None) + + def merge_rounds_std(rs, rounds): + loaded = {} + for r in rounds: + la, pa, sa, psa = rs[r].la, rs[r].pa, rs[r].sa, rs[r].psa + lsc = [] + for k0, n in lse_chunks: + if fx.const_expr(n == 1): + lsc.append(fx.Float32(pld(la + fx.Int64(4 * k0), 1))) + else: + vv_ = fx.Vector(pld(la + fx.Int64(4 * k0), n)) + lsc += [fx.Float32(vv_[jx]) for jx in range(n)] + pscl = None + if fx.const_expr(PF16): + pscl = [] + for k0, n in lse_chunks: + if fx.const_expr(n == 1): + pscl.append( + fx.Float32(pld(psa + fx.Int64(4 * k0), 1)) + ) + else: + vv_ = fx.Vector(pld(psa + fx.Int64(4 * k0), n)) + pscl += [fx.Float32(vv_[jx]) for jx in range(n)] + pvs = [ + fx.Vector(pld16(pa[sidx])).to(fx.Float32) + for sidx in range(S) + ] + elif fx.const_expr(ADAPT): + pvs = [ + fx.Vector( + pld( + ((tps * fx.Int32(sidx)) < ntiles).select( + pa[sidx], pa[0] + ), + EPT, + ) + ) + for sidx in range(S) + ] + else: + pvs = [fx.Vector(pld(pa[sidx], EPT)) for sidx in range(S)] + snk = fx.Float32(gload(sa, 1)) + loaded[r] = (lsc, pvs, snk, pscl) + fx.rocdl.sched_barrier(0) + for r in rounds: + lsc, pvs, snk, pscl = loaded[r] + snk = snk * fx.Float32(LOG2E) + gmax = snk + lv, vsl = [], [] + for sidx in fx.range_constexpr(S): + vs = (tps * fx.Int32(sidx)) < ntiles + vsl.append(vs) + lv.append(vs.select(lsc[sidx], ninf)) + gmax = gmax.maximumf(lv[sidx]) + den = exp2(snk - gmax) + zv = fx.Vector.filled(EPT, 0.0, fx.Float32) + acc = zv + for sidx in fx.range_constexpr(S): + w = exp2(lv[sidx] - gmax) + den = den + w + wp = w * pscl[sidx] if PF16 else w + acc = acc + vsl[sidx].select(fx.Vector(pvs[sidx]) * wp, zv) + merge_finish(r, rs, acc, den, lv) + + merge_rounds = merge_rounds_sk if SK else merge_rounds_std + + if fx.const_expr(RQM): + pass + elif t0 < t1: + for rb in fx.range_constexpr(NRBE): + if fx.const_expr(RQ): + rloc = wv * fx.Int32(32) + r32 + elif fx.const_expr(RD): + rloc = fx.Int32(32 * rb) + r32 + else: + rloc = fx.Int32(16 * rb) + m + has = l_fin[rb] > fx.Float32(0.0) + inv_l = has.select( + fx.Float32(1.0) / l_fin[rb].maximumf(fx.Float32(1e-30)), + fx.Float32(0.0), + ) + lse2 = has.select( + m_fin[rb] + + fly_math.log2(l_fin[rb].maximumf(fx.Float32(1e-30))), + ninf, + ) + rec = ( + fx.Int64(group) * fx.Int64(S) + fx.Int64(split) + ) * fx.Int64(ROWS) + fx.Int64(rloc) + if fx.const_expr(RQ): + lds_barrier() + PQ_P = 528 + pq_wb = wv * fx.Int32(32 * PQ_P) + pq_rec0 = ( + fx.Int64(group) * fx.Int64(S) + fx.Int64(split) + ) * fx.Int64(ROWS) + fx.Int64(wv * fx.Int32(32)) + for qd in fx.range_constexpr(4): + for jj in fx.range_constexpr(4): + for qq in fx.range_constexpr(4): + nt = 4 * (4 * qd + jj) + qq + _llvm_d.StoreOp( + _arith.unwrap(acc_fin[rb][nt] * inv_l), + fx.to_llvm_ptr( + p_rq + + ( + pq_wb + + r32 * fx.Int32(PQ_P) + + fx.Int32(4 * (32 * jj + 8 * qq)) + + hq * fx.Int32(16) + ) + ), + alignment=16, + ) + for pq_i2 in fx.range_constexpr(16): + pq_rr = fx.Int32(2 * pq_i2) + hq + pq_row = lds_rd( + pq_wb + + pq_rr * fx.Int32(PQ_P) + + r32 * fx.Int32(16), + fx.Vector.make_type(4, fx.Int32), + 16, + ) + pst( + part_base + + ( + (pq_rec0 + fx.Int64(pq_rr)) * fx.Int64(DV) + + fx.Int64( + fx.Int32(128 * qd) + r32 * fx.Int32(4) + ) + ) + * fx.Int64(4), + pq_row, + 16, + ) + if fx.const_expr(qd < 3): + _llvm_d.InlineAsmOp( + None, + [], + "s_waitcnt lgkmcnt(0)", + "~{memory}", + has_side_effects=True, + ) + elif fx.const_expr(Q16): + Q_PQP = 528 + qp_wb = wv * fx.Int32(ROWS * Q_PQP) + qp_rec0 = ( + fx.Int64(group) * fx.Int64(S) + fx.Int64(split) + ) * fx.Int64(ROWS) + Q_PQP = 272 + qp_wb = wv * fx.Int32(ROWS * Q_PQP) + if wv < fx.Int32(4): + pv_ = [acc_fin[rb][nt] * inv_l for nt in range(NACCW)] + + amx = fx.Float32(0.0) + for nt in fx.range_constexpr(NACCW): + for i in fx.range_constexpr(4): + amx = maxnum_f32(amx, fabs_f32(pv_[nt][i])) + amx = plswap_f(amx, 32, maxnum_f32) + ex_ = amx.bitcast(fx.Int32) >> fx.Int32(23) + kx = fx.Int32(141) - ex_ + kx = smax_i32( + smin_i32(kx, fx.Int32(126)), fx.Int32(-126) + ) + sdn = ((kx + fx.Int32(127)) << fx.Int32(23)).bitcast( + fx.Float32 + ) + sup = ((fx.Int32(127) - kx) << fx.Int32(23)).bitcast( + fx.Float32 + ) + if hq == zero_i: + pst( + psc_base + + ( + ( + fx.Int64(group) * fx.Int64(ROWS) + + fx.Int64(rloc) + ) + * fx.Int64(4 * SP) + + fx.Int64( + wv * fx.Int32(SP) + fx.Int32(split) + ) + ) + * fx.Int64(4), + sup, + 4, + ) + for nt in fx.range_constexpr(NACCW): + fx.rocdl.sched_barrier(0) + v_ = (pv_[nt] * sdn).to(fx.Float16) + _llvm_d.StoreOp( + _arith.unwrap(v_.bitcast(fx.Int32)), + fx.to_llvm_ptr( + p_rq + + ( + qp_wb + + rloc * fx.Int32(Q_PQP) + + fx.Int32( + 2 * (32 * (nt // 4) + 8 * (nt % 4)) + ) + + hq * fx.Int32(8) + ) + ), + alignment=8, + ) + if fx.const_expr(rb == NRBE - 1): + for qp_k in fx.range_constexpr(ROWS // 4): + qp_rr = fx.Int32(4 * qp_k) + g + qp_row = lds_rd( + qp_wb + + qp_rr * fx.Int32(Q_PQP) + + m * fx.Int32(16), + fx.Vector.make_type(4, fx.Int32), + 16, + ) + pst( + part_base + + ( + (qp_rec0 + fx.Int64(qp_rr)) + * fx.Int64(DV) + + fx.Int64( + wv * fx.Int32(128) + m * fx.Int32(8) + ) + ) + * fx.Int64(2), + qp_row, + 16, + ) + else: + for nt in fx.range_constexpr(NACCW): + pst( + part_base + + (rec * fx.Int64(DV) + fx.Int64(dv_of(nt))) + * fx.Int64(4), + acc_fin[rb][nt] * inv_l, + 16, + ) + if fx.const_expr(RQ): + lse_w = hq == zero_i + elif fx.const_expr(RD): + lse_w = (hq == zero_i) & (wv == zero_i) + else: + lse_w = (g == zero_i) & (wl == zero_i) + if lse_w: + pst( + lse_base + + ( + (fx.Int64(group) * fx.Int64(ROWS) + fx.Int64(rloc)) + * fx.Int64(SP) + + fx.Int64(split) + ) + * fx.Int64(4), + lse2, + 4, + ) + + if fx.const_expr(not RQM): + own = slice_addrs(split, True) + if fx.const_expr(OWN): + if wave == zero_i: + tst0 = spin_tickets_state( + tk_lane, tkv_own, START_POLLS, myx + fx.Int32(1) + ) + if lane == zero_i: + lds_st(p_flag, fx.Int32(2), tst0) + lds_barrier() + tst_c = fx.Int32(fx.ptr_load(p_flag + fx.Int32(2))) + leave = (tst_c & fx.Int32(1)) == zero_i + rq_store(own_ip, own_lse, own_acc, leave) + tst_mask = sk_last.select(fx.Int32(-1), zero_i) + wb_on = sk_wt == false_ + # arrive: partials stored (vmcnt 0) before the tickets are judged. Stay only if every + # sibling started; release by L2 write-back unless all siblings share this XCD (or + # the partials were written through), then one returning atomic add of the bits + fx.rocdl.s_waitcnt(vmcnt=0) + fx.gpu.barrier() + if wave == zero_i: + if fx.const_expr(OWN): + tst = fx.Int32(fx.ptr_load(p_flag + fx.Int32(2))) + else: + tst = spin_tickets_state( + tk_lane, tkv, START_POLLS, myx + fx.Int32(1) + ) + if fx.const_expr(SK): + tst = tst & tst_mask + stay = (tst & fx.Int32(1)) != zero_i + col = (tst & fx.Int32(2)) != zero_i + av = fx.Int64(0) + wb = false_ if WTP else (col == false_) & wb_on + if lane == zero_i: + if wb: + cu.fence_agent_release() + bit = fx.Int64(1) << fx.Int64(split) + inc = bit + stay.select(bit << fx.Int64(32), fx.Int64(0)) + av = rmw_add64_agent(slot + fx.Int64(A_OFF), inc) + inc + av = fx.Int64( + fx.rocdl.readfirstlane(T.i64, fx.Int64(av).ir_value()) + ) + if fx.const_expr(DYN): + fullv = (fx.Int64(1) << fx.Int64(ns)) - fx.Int64(1) + last = fx.Int32(av & fullv) == fx.Int32(fullv) + else: + fullv = fx.Int64(FULL) + last = fx.Int32(av & fx.Int64(FULL)) == fx.Int32(FULL32) + # the last arriver resets the slot for the next launch (graph replay) + if last: + if lane < fx.Int32(S): + st_raw(tk_lane, zero_i, 4) + if lane == zero_i: + st_raw(slot + fx.Int64(A_OFF), fx.Int64(0), 8) + if lane == zero_i: + if stay & (last == false_): + spin_all_arrived( + slot + fx.Int64(A_OFF), + fullv, + fx.Int64(1) << fx.Int64(split), + ) + flv = ( + stay.select(fx.Int32(1), zero_i) + | last.select(fx.Int32(2), zero_i) + | (stay & col).select(fx.Int32(4), zero_i) + ) + lds_st(p_flag, zero_i, flv) + lds_st(p_flag, fx.Int32(1), fx.Int32(av >> fx.Int64(32))) + fx.gpu.barrier() + fl = fx.Int32(fx.ptr_load(p_flag)) + smask = fx.Int32(fx.ptr_load(p_flag + fx.Int32(1))) + if (fl & fx.Int32(3)) != zero_i: + # acquire: L1 invalidate when colocated, else agent-scope + inv_all = false_ if WTP else ((fl & fx.Int32(4)) == zero_i) + if inv_all: + cu.fence_agent_acquire() + else: + inv_l1() + if fx.const_expr(OWN): + merge_own(split, True) + else: + merge_slice(own) + if (fl & fx.Int32(2)) != zero_i: + for jsl in range( + zero_i, ns if DYN else fx.Int32(S), fx.Int32(1) + ): + j = fx.Int32(jsl) + if (j != split) & (((smask >> j) & fx.Int32(1)) == zero_i): + if fx.const_expr(OWN): + merge_own(j, False) + else: + merge_slice(slice_addrs(j, False)) + + def epilogue(): + if fx.const_expr(not MERGED): + if fx.const_expr(not PV16R): + direct_epi() + elif fx.const_expr(DYN or SHORT): + if ns == fx.Int32(1): + if split == zero_i: + direct_epi() + else: + if live: + merged_epi() + else: + merged_epi() + + epilogue() + + return types.SimpleNamespace( + emit=emit, + map_bid=map_bid, + Smem=Smem, + NT=NT, + NRG=NRG, + S=S, + name=name, + agpr=RQ, + ) + + +# the traced bodies live in helpers the compile cache does not see: key on their source +# (needs the .py files, which aiter installs) +def _source_key(): + h = hashlib.sha256() + d = os.path.dirname(os.path.abspath(__file__)) + for f in ("mla_v4_decode_gfx950.py", "mla_v4_decode_common.py"): + with open(os.path.join(d, f), "rb") as fh: + h.update(fh.read()) + return h.hexdigest()[:16] + + +_SRC_KEY = _source_key() + + +@functools.cache +def build_mla_v4_decode(cfg: V4Cfg): + main = _emitter(cfg) + IDLE = cfg.IDLE + one = _emitter(replace(cfg, S=1, PF16=False, IDLE=())) if IDLE else main + emit_one = one.emit + subs = [] + wsp = cfg.ws_seq() + for keys, sc in cfg.SUB: + subs.append((keys, sc, _emitter(sc, wsp))) + g1, b1 = sc.ws_seq() + wsp = (wsp[0] + g1, wsp[1] + b1) + emit_main = main.emit + map_bid = main.map_bid + NT = main.NT + NRG, S = main.NRG, main.S + NSUB = len(subs) + CPS = cfg.cps + SUB_K = [k for k, _, _ in subs] + SUB_S = [sc.S for _, sc, _ in subs] + SUB_C = [sc.NRG * sc.S for _, sc, _ in subs] + SUB_W = [em.NT // WAVE for _, _, em in subs] + SUB_EMIT = [em.emit for _, _, em in subs] + SPREAD = cfg.SPREAD + SKG = cfg.SK + SKD = cfg.SKD + MINT = cfg.MINT + SKI = -(-SKG // WAVE) + SKM = 2 # key-balanced map only if the longest split > 1.25 runs + SKM tiles + SKT = cfg.SKT + SKO = 256 # keys charged per stream for a segment's fixed cost + SKMEAN = cfg.SKMEAN + SKTL = cfg.KT + SKSH = SKTL.bit_length() - 1 + QSEQ = cfg.EPI == "invrope_mxfp8" and cfg.MSQ == 1 + name = ( + main.name + + "".join(f"_sub{k}r{sc.RB}s{sc.S}" for k, sc in cfg.SUB) + + ("_spr" if SPREAD else "") + + ("_idle{}_{}_{}_{}".format(*IDLE) if IDLE else "") + ) + if NSUB == 0 and not IDLE: + Smem = main.Smem + else: + fields = {"lng": main.Smem} + fields.update({f"sb{i}": em.Smem for i, (_, _, em) in enumerate(subs)}) + if IDLE: + fields["one"] = one.Smem + Smem = fx.union(type("Smem", (), {"__annotations__": fields})) + + CFG_KEY = repr(cfg) + _SRC_KEY + + @flyc.kernel(name=name, known_block_size=[NT, 1, 1]) + def kernel( + nseq: fx.Int32, + num_rows: fx.Int32, + indptr_ptr: fx.Pointer, + idx_ptr: fx.Pointer, + qo_ptr: fx.Pointer, + q_ptr: fx.Pointer, + kv_ptr: fx.Pointer, + kvr_ptr: fx.Pointer, + qr_ptr: fx.Pointer, + pos_ptr: fx.Pointer, + ws_ptr: fx.Pointer, + sink_ptr: fx.Pointer, + out_ptr: fx.Pointer, + xs_ptr: fx.Pointer, + fr_ptr: fx.Pointer, + out_s0: fx.Int32, + out_s1: fx.Int32, + hdr_ptr: fx.Pointer, + bnd_ptr: fx.Pointer, + ): + # flydsl's compile cache keys on the source and closure values only: reading + # CFG_KEY gives every variant (and source revision) its own binary + _variant = CFG_KEY + args = ( + nseq, + num_rows, + indptr_ptr, + idx_ptr, + q_ptr, + kv_ptr, + kvr_ptr, + qr_ptr, + qo_ptr, + pos_ptr, + ws_ptr, + sink_ptr, + out_ptr, + xs_ptr, + fr_ptr, + out_s0, + out_s1, + hdr_ptr, + bnd_ptr, + ) + if fx.const_expr(SKG): + lds = fx.SharedAllocator().allocate(Smem).peek() + zero = fx.Int32(0) + one = fx.Int32(1) + lane = fx.Int32(fx.thread_idx.x) & fx.Int32(WAVE - 1) + false_ = one == zero + true_ = one != zero + c = fx.Int32(fx.block_idx.x) + if fx.const_expr(cfg.TAIL): + seq_s, split_s = _tail_map(c, nseq, 1, SKD) + else: + bj = c >> fx.Int32(3) + gps = (nseq + fx.Int32(7)) >> fx.Int32(3) + split_s = udiv(bj, gps) + seq_s = ((bj - split_s * gps) << fx.Int32(3)) | (c & fx.Int32(7)) + ok_s = seq_s < nseq + seq_sc = ok_s.select(seq_s, zero) + bounds = [] + for k in fx.range_constexpr(SKI): + bk = lane + fx.Int32(WAVE * k) + ok = bk < nseq + bc = ok.select(bk, zero) + a0 = fx.Int32(fx.ptr_load(indptr_ptr + fx.Int64(bc))) + a1 = fx.Int32(fx.ptr_load(indptr_ptr + fx.Int64(bc) + fx.Int64(1))) + bounds.append((bk, ok, a0, a1)) + # (one round trip with the batch bounds) + ptrs = (indptr_ptr,) if QSEQ else (indptr_ptr, qo_ptr) + md = [_sload(p, seq_sc + fx.Int32(i)) for p in ptrs for i in (0, 1)] + md = _pin_s(md + [_sload(indptr_ptr, zero), _sload(indptr_ptr, nseq)]) + kv0_s, kvlen_s = md[0], md[1] - md[0] + q0_s, qlen_s = (seq_sc, one) if QSEQ else (md[2], md[3] - md[2]) + base = md[-2] + nkeys = md[-1] - base + nseq * fx.Int32(SKO) + share = smax_i32(_div_const(nkeys + fx.Int32(SKG - 1), SKG), fx.Int32(SKTL)) + share_t = (share + fx.Int32(SKTL - 1)) >> fx.Int32(SKSH) + mint = fx.Int32(MINT) + if fx.const_expr(SKMEAN): + kts = smax_i32(nseq, one) * fx.Int32(SKTL) + mint = smax_i32( + mint, udiv(nkeys - nseq * fx.Int32(SKO) + kts - one, kts) + ) + b_lo = zero + b_end = zero + run = share + nrun = one + + def owners(a, n, run, nrun): + f = smin_i32(udiv(a, run), nrun - one) + nt = (n + fx.Int32(SKTL - 1)) >> fx.Int32(SKSH) + la = udiv(a + (nt - one) * fx.Int32(SKTL), run) + return f, (n > zero).select(la, f), nt + + seqs = [] + nmax = zero + for bk, ok, a0, a1 in bounds: + n = ok.select(a1 - a0, zero) + seqs.append((ok, a0 - base + (bk + one) * fx.Int32(SKO), n)) + nmax = smax_i32(nmax, n) + nmax = butterfly_i32(nmax, WAVE, smax_i32) + ntmax = (nmax + fx.Int32(SKTL - 1)) >> fx.Int32(SKSH) + tpmax = _div_const(ntmax + fx.Int32(SKD - 1), SKD) + if fx.const_expr(SKMEAN): + tpmax = smin_i32(ntmax, smax_i32(tpmax, mint)) + # short runs: a split of > SKT tiles, >= 2 runs and >= 4 shares (skew) + runt = smax_i32(share_t, _div_const(ntmax + fx.Int32(S - 2), S - 1)) + skew = tpmax >= smax_i32(runt * fx.Int32(2), share_t * fx.Int32(4)) + use_sk = (share_t >= fx.Int32(SKT)) & ( + tpmax * fx.Int32(4) > share_t * fx.Int32(5) + fx.Int32(4 * SKM) + ) | (tpmax > fx.Int32(SKT)) & skew + if fx.const_expr(SKMEAN): + use_sk = use_sk & ( + (ntmax * fx.Int32(4) > mint * fx.Int32(5)) + | (share_t >= fx.Int32(2 * SKT)) + ) + if use_sk: + run = smax_i32(share, _div_const(nmax + fx.Int32(S - 2), S - 1)) + nrun = smax_i32(udiv(nkeys + run - one, run), one) + cnt = zero + for ok, a, n in seqs: + f, la, _ = owners(a, n, run, nrun) + cnt = cnt + (ok & (la < c)).select(one, zero) + cnt = cnt + (ok & (f <= c)).select(fx.Int32(1 << 16), zero) + cnt = butterfly_i32(cnt, WAVE, _fadd) + b_lo = cnt & fx.Int32(0xFFFF) + b_end = cnt >> fx.Int32(16) + if use_sk: + for bi in range(b_lo, b_end, one): + b = fx.Int32(bi) + kv0 = fx.Int32(fx.ptr_load(indptr_ptr + fx.Int64(b))) + kvlen = ( + fx.Int32(fx.ptr_load(indptr_ptr + fx.Int64(b) + fx.Int64(1))) + - kv0 + ) + q0 = fx.Int32(fx.ptr_load(qo_ptr + fx.Int64(b))) + qlen = ( + fx.Int32(fx.ptr_load(qo_ptr + fx.Int64(b) + fx.Int64(1))) - q0 + ) + a = kv0 - base + (b + one) * fx.Int32(SKO) + f, la, nt = owners(a, kvlen, run, nrun) + lo = c * run - a + hi = lo + run + t0 = (lo > zero).select( + (lo + fx.Int32(SKTL - 1)) >> fx.Int32(SKSH), zero + ) + t1 = (hi > zero).select( + (hi + fx.Int32(SKTL - 1)) >> fx.Int32(SKSH), zero + ) + ns = la - f + one + emit_main( + *args, + lds, + b, + zero, + c - f, + (kv0, kvlen), + ( + t0, + smin_i32(t1, nt), + ns, + b == b_end - one, + true_, + (q0, qlen), + one, + ns, + ), + ) + fx.gpu.barrier() + else: + nt = (kvlen_s + fx.Int32(SKTL - 1)) >> fx.Int32(SKSH) + tps = smax_i32(_div_const(nt + fx.Int32(SKD - 1), SKD), mint) + if fx.const_expr(cfg.SHORT): + tps = (nt <= fx.Int32(cfg.SHORT)).select(nt, tps) + t0 = split_s * tps + ns = (nt > tps).select(fx.Int32(SKD), one) + wt_s = false_ + if fx.const_expr(cfg.TAIL): + wt_s = seq_s >= (nseq & fx.Int32(-8)) + if ok_s & (split_s < ns): + emit_main( + *args, + lds, + seq_s, + zero, + split_s, + (kv0_s, kvlen_s), + ( + t0, + smin_i32(t0 + tps, nt), + ns, + true_, + wt_s, + (q0_s, qlen_s), + tps, + nt, + ), + ) + elif fx.const_expr(IDLE): + st = fx.SharedAllocator().allocate(Smem) + lds_2, lds_1 = st.lng.peek(), st.one.peek() + zero = fx.Int32(0) + c = fx.Int32(fx.block_idx.x) + xcd = c & fx.Int32(7) + gpx = ((nseq + fx.Int32(7)) >> fx.Int32(3)) * fx.Int32(NRG) + ext = (c >> fx.Int32(3)) >= gpx + k = (c >> fx.Int32(3)) - ext.select(gpx, zero) + rg = _rem_const(k, NRG) + seq = (_div_const(k, NRG) << fx.Int32(3)) | xcd + tl_lane = fx.Int32(fx.thread_idx.x) & fx.Int32(WAVE - 1) + if ext: + # splits 1.. of the stream of rank k / NRG / (S - 1) on this XCD (none: + # past nseq) + j, ok, rk = xcd_ranks(indptr_ptr, nseq, xcd, tl_lane, IDLE[0] // NRG) + hit = ok & (rk == _div_const(_div_const(k, NRG), S - 1)) + j1 = butterfly_i32(hit.select(j + fx.Int32(1), zero), WAVE, _fadd) + j1 = fx.Int32(fx.rocdl.readfirstlane(T.i32, j1.ir_value())) + seq = (j1 == zero).select(nseq, j1 - fx.Int32(1)) + seq = fx.Int32(fx.rocdl.readfirstlane(T.i32, seq.ir_value())) + seq_c = (seq < nseq).select(seq, zero) + ptrs = (indptr_ptr,) if QSEQ else (qo_ptr, indptr_ptr) + md = [_sload(p, seq_c + fx.Int32(i)) for p in ptrs for i in (0, 1)] + md = _pin_s(md) + kvall = _sload(indptr_ptr, nseq) - _sload(indptr_ptr, zero) + kvm = (md[-2], md[-1] - md[-2]) + (() if QSEQ else (md[0], md[1] - md[0])) + nt = (kvm[1] + fx.Int32(SKTL - 1)) >> fx.Int32(SKSH) + # split: longer than the mean and f tiles, splits 1.. on CTAs of this XCD + two = (seq < nseq) & (nt > fx.Int32(IDLE[2])) + two = two & ( + fx.Int64(nt - fx.Int32(1)) * fx.Int64(nseq * fx.Int32(SKTL)) + >= fx.Int64(kvall) + ) + if two: + _, _, rk = xcd_ranks(indptr_ptr, nseq, xcd, tl_lane, IDLE[0] // NRG) + rk = fx.Int32( + fx.rocdl.readlane( + T.i32, rk.ir_value(), (seq >> fx.Int32(3)).ir_value() + ) + ) + pos = gpx + (rk + fx.Int32(1)) * fx.Int32(NRG * (S - 1)) + two = (pos <= fx.Int32(IDLE[0])) | ( + (pos <= fx.Int32(IDLE[0] + IDLE[1])) & (nt > fx.Int32(IDLE[3])) + ) + if two: + split = fx.Int32(1) + _rem_const(_div_const(k, NRG), S - 1) + emit_main(*args, lds_2, seq, rg, ext.select(split, zero), kvm) + else: + if (c >> fx.Int32(3)) < gpx: + emit_one(*args, lds_1, seq, rg, zero, kvm) + elif fx.const_expr(NSUB == 0): + seq, rg, split = map_bid(fx.Int32(fx.block_idx.x), nseq) + emit_main(*args, fx.SharedAllocator().allocate(Smem).peek(), seq, rg, split) + else: + st = fx.SharedAllocator().allocate(Smem) + lds_m = st.lng.peek() + lds_s = [getattr(st, f"sb{i}").peek() for i in range(NSUB)] + bid = fx.Int32(fx.block_idx.x) + if fx.const_expr(SPREAD): + seq = _div_const(bid, CPS) + c = bid - seq * fx.Int32(CPS) + elif fx.const_expr(cfg.TAIL): + seq, c = _tail_map(bid, nseq, 1, CPS) + else: + bj = bid >> fx.Int32(3) + sg = _div_const(bj, CPS) + c = bj - sg * fx.Int32(CPS) + seq = (sg << fx.Int32(3)) | (bid & fx.Int32(7)) + seq_c = (seq < nseq).select(seq, fx.Int32(0)) + ptrs = (indptr_ptr,) if QSEQ else (indptr_ptr, qo_ptr) + md = [_sload(ptr, seq_c + fx.Int32(j)) for ptr in ptrs for j in range(2)] + kvlen = md[1] - md[0] + kvm = (md[0], kvlen) + if fx.const_expr(not QSEQ): + kvm = kvm + (md[2], md[3] - md[2]) + wv = fx.Int32( + fx.rocdl.readfirstlane( + T.i32, (fx.Int32(fx.thread_idx.x) >> fx.Int32(6)).ir_value() + ) + ) + + def run(emit_b, lds_b, s_b, cps_b, w_b): + if c < fx.Int32(cps_b): + rgi = _div_const(c, s_b) + spi = c - rgi * fx.Int32(s_b) + if fx.const_expr(w_b * WAVE == NT): + emit_b(*args, lds_b, seq, rgi, spi, kvm) + else: + if wv < fx.Int32(w_b): + emit_b(*args, lds_b, seq, rgi, spi, kvm) + + def body(i): + if fx.const_expr(i == NSUB): + run(emit_main, lds_m, S, NRG * S, NT // WAVE) + else: + if kvlen <= fx.Int32(SUB_K[i]): + run(SUB_EMIT[i], lds_s[i], SUB_S[i], SUB_C[i], SUB_W[i]) + else: + body(i + 1) + + body(0) + + def _kattrs(): + if not main.agpr: + return {} + va = {} + va["passthrough"] = _ir.ArrayAttr.get( + [ + _ir.ArrayAttr.get( + [ + _ir.StringAttr.get("amdgpu-agpr-alloc"), + _ir.StringAttr.get("256"), + ] + ) + ] + ) + return {"value_attrs": va} + + @flyc.jit + def launch( + q_ptr: fx.Pointer, + qr_ptr: fx.Pointer, + kv_ptr: fx.Pointer, + kvr_ptr: fx.Pointer, + qo_ptr: fx.Pointer, + indptr_ptr: fx.Pointer, + idx_ptr: fx.Pointer, + sink_ptr: fx.Pointer, + out_ptr: fx.Pointer, + xs_ptr: fx.Pointer, + pos_ptr: fx.Pointer, + fr_ptr: fx.Pointer, + ws_ptr: fx.Pointer, + bnd_ptr: fx.Pointer, + hdr_ptr: fx.Pointer, + nseq: fx.Int32, + num_rows: fx.Int32, + out_s0: fx.Int32, + out_s1: fx.Int32, + stream: fx.Stream, + ): + if fx.const_expr(SKG): + nstd = ((nseq + fx.Int32(7)) // fx.Int32(8)) * fx.Int32(8 * cfg.SKD) + if fx.const_expr(cfg.TAIL): + nstd = nseq * fx.Int32(cfg.SKD) + nblk = (nstd > fx.Int32(SKG)).select(nstd, fx.Int32(SKG)) + elif fx.const_expr(IDLE): + nstd = ((nseq + fx.Int32(7)) // fx.Int32(8)) * fx.Int32(8 * NRG) + ntl = fx.Int32(8 * (IDLE[0] + IDLE[1])) + nblk = (nstd > ntl).select(nstd, ntl) + elif fx.const_expr(SPREAD or cfg.TAIL): + nblk = nseq * fx.Int32(CPS) + else: + nblk = ((nseq + fx.Int32(7)) // fx.Int32(8)) * fx.Int32(8 * CPS) + kernel( + nseq, + num_rows, + indptr_ptr, + idx_ptr, + qo_ptr, + q_ptr, + kv_ptr, + kvr_ptr, + qr_ptr, + pos_ptr, + ws_ptr, + sink_ptr, + out_ptr, + xs_ptr, + fr_ptr, + out_s0, + out_s1, + hdr_ptr, + bnd_ptr, + **_kattrs(), + ).launch( + grid=(nblk, 1, 1), + block=(NT, 1, 1), + stream=stream, + ) + + llvm_opts = { + "amdgpu-kernarg-preload": True, + "amdgpu-kernarg-preload-count": 32, + } + launch.compile_hints = {"llvm_options": llvm_opts} + return launch diff --git a/python/sglang/kernels/ops/attention/dsv4/flydsl_v4nm/mla_v4_decode.py b/python/sglang/kernels/ops/attention/dsv4/flydsl_v4nm/mla_v4_decode.py new file mode 100644 index 0000000000..4821379f2d --- /dev/null +++ b/python/sglang/kernels/ops/attention/dsv4/flydsl_v4nm/mla_v4_decode.py @@ -0,0 +1,718 @@ +# SPDX-License-Identifier: MIT +# Copyright (C) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. +# +# Vendored from ROCm/aiter#5882 (https://github.com/ROCm/aiter/pull/5882), +# commit 03359c43df (03359c43df40abf91ac38a2040c9b8e284d92297), +# file aiter/ops/flydsl/mla_v4_decode.py. +# Unchanged except for import paths and sglang formatting (isort, ruff format). +# TODO(aiter#5882): remove once ROCm/aiter#5882 is in the aiter pin + +"""FlyDSL DeepSeek-V4 fp8 MLA decode (v4 nm contract) on gfx950. + +flydsl_mla_decode_fwd_v4_nm: drop-in for aiter.mla.mla_decode_fwd_v4_nm (bf16 output). +flydsl_mla_v4_decode_fused: the same attention + inverse RoPE + wo_a mxfp8 quant. + +The split plan is keyed by the batch size, head count, max_seqlen_q and a stream hint, +never by kv lengths (graph-capture safe). The merge scratch is owned here, one per +(device, stream). +""" + +from __future__ import annotations + +import dataclasses +from functools import cache + +import flydsl.expr as fx +import torch +from aiter.jit.utils.chip_info import get_gfx +from aiter.ops.flydsl.kernels.tensor_shim import _run_compiled, ptr_arg + +from .kernels.mla_v4_decode_gfx950 import ( + DV, + MAX_GROUPS, + MAX_S, + ROPE, + ROWB, + WS_HDR, + V4Cfg, + build_mla_v4_decode, +) +from .mla_v4_grouped import GROUPED_MAX_MSQ + +__all__ = [ + "flydsl_mla_decode_fwd_v4_nm", + "flydsl_mla_v4_decode_fused", + "flydsl_mla_v4_decode_supported", + "flydsl_mla_v4_grouped_pays", +] + +# (H, max_seqlen_q) pairs the asm v4 nm kernel ships +SUPPORTED = ((16, 1), (16, 2), (16, 4), (32, 1), (64, 1), (128, 1)) + +# Split plans measured on MI355X (cold caches, vs the best asm split count + merge). +# H16 msq1 with a stream hint: T_pad upper bound -> (CSA, HCA) splits (HCA past +# T_pad 40: _plan_h16's key-balanced grid) +_POLICY_H16 = { + 8: (18, 18), + 16: (12, 12), + 24: (9, 9), + 32: (8, 8), + 40: (6, 6), + 48: (4, 4), + 64: (4, 4), + 80: (3, 3), + 128: (2, 2), + 160: (3, 1), + 256: (1, 1), +} +# below _POLICY_Q16: B upper bound -> (16-row blocks per CTA, splits) of K SPREAD plans +_POLICY_WIDE = { + (16, 2): ((1, 1, 32), (None, 1, 16)), + (16, 4): ((None, 1, 16),), + (32, 1): ((1, 1, 24), (None, 1, 16)), + (64, 1): ((None, 1, 16),), + (128, 1): ((None, 1, 16),), +} +# stream-length bodies (V4Cfg.SUB): B upper bound -> longest short stream (K, S = 1); +# (H, msq) -> ((B max, B min, ((RB, S, longest stream), ...)), ...) mid-length streams +_SHORT_KEYS = ((8, 192), (16, 256), (64, 768), (128, 256)) +# Q16 mains: 32-row ones take the short body up to _Q16_SHORT_KEYS keys; 64-row ones +# past B 16 at S >= 3 run streams of <= max(_Q16_RB2_KEYS / S, _Q16_RB2_MIN) keys on +# unsplit 32-row Q16 CTAs (at S = 2 the joint kernel spills: long streams pay more) +_Q16_SHORT_KEYS = 160 +_Q16_RB2_KEYS, _Q16_RB2_MIN = 2304, 448 +_POLICY_MID = { + (16, 2): ((1, 1, ((1, 16, 2048),)), (16, 9, ((1, 8, 4096),))), + (16, 4): ((8, 5, ((1, 8, 3072),)), (16, 9, ((1, 4, 2048),))), + (32, 1): ((16, 9, ((1, 8, 4096),)),), + (64, 1): ((8, 5, ((1, 8, 3072),)), (16, 9, ((1, 4, 2048),))), +} +# smallest B of each pair on Q16 CTAs +_POLICY_Q16 = {(16, 2): 9, (16, 4): 5, (32, 1): 9, (64, 1): 5, (128, 1): 2} +# grids a little past half the GPU run 2 splits: largest grid, in 1/32 of the CUs, for +# Q16 (RB, NRG) and for the H16 K layout +_PAST_HALF = {(4, 1): 18, (2, 1): 19} +_PAST_HALF_K = 19 +# Q16 split count -> streams of at most this many 32-key tiles run unsplit (SHORT) +_Q16_SHORT_TILES = {2: 24, 3: 10} +# unsplit Q16 grids of fewer CTAs than CUs: split 1 of the longest streams of > _IDLE_F +# tiles on the idle CUs (V4Cfg.IDLE), past them on _IDLE_X2 more CTAs per XCD for +# streams of > _IDLE_W tiles; larger grids of two row groups (up to 512 sequences): +# splits 1-3 of streams of > _IDLE_WL tiles on _IDLE_XL CTAs per XCD past the grid +_IDLE_X2, _IDLE_F, _IDLE_W = 4, 32, 96 +_IDLE_XL, _IDLE_WL = 16, 64 +# PV16R key window: (lo, hi at B <= CUs, hi above) +_PV16_LO, _PV16_HI = 256, (4096, 2048) +# grouped HCA: streams of <= 17 32-key tiles on the K body, longer ones on RQ +_GROUPED_RQ_TILES = 18 +# H16 bf16 without a hint: most splits per sequence; longest unsplit stream; shortest +# key-balanced SK run in 64-key tiles (grid of one, two CTAs per CU) +_H16_MAX_S = 18 +_H16_SHORT_KEYS = 128 +_H16_SKT = (8, 12) +# SK grids of at most this many splits per sequence: V4Cfg.TAIL where it adds some +_H16_TAIL_SKD = 8 +# 64-row Q16 with fewer splits than this per sequence: V4Cfg.TAIL where it adds some +_TAIL_S = 16 + + +@cache +def _num_cus(device_index: int) -> int: + return torch.cuda.get_device_properties(device_index).multi_processor_count + + +def flydsl_mla_v4_decode_supported( + num_heads: int, max_seqlen_q: int = 1, grouped: bool = False +) -> bool: + """True if the FlyDSL v4 decode covers (num_heads, max_seqlen_q) on this GPU; + grouped: with q_kv_bounds (H = 16, max_seqlen_q <= 8).""" + try: + if get_gfx() != "gfx950": + return False + except Exception: # noqa: BLE001 + return False + if grouped: + return num_heads == 16 and 1 <= max_seqlen_q <= GROUPED_MAX_MSQ + return (num_heads, max_seqlen_q) in SUPPORTED + + +def flydsl_mla_v4_grouped_pays( + num_heads: int, + num_draft: int, + num_requests: int, + hint: str | None = "hca", + epi: str = "invrope_mxfp8", +) -> bool: + """Whether a DSpark verify step should share one stream per request (grouped + q_kv_bounds) on its HCA / SWA-only layers (MI355X, DSv4-Pro verify step).""" + if num_heads != 16 or not 1 <= num_draft <= GROUPED_MAX_MSQ: + return False + if epi == "bf16" and hint in (None, "hca"): + return num_requests >= 32 + return num_requests >= 64 + + +def _plan( + H: int, + msq: int, + B: int, + epi: str = "bf16", + hint: str | None = None, + cus: int = 256, + grouped: bool = False, +) -> V4Cfg: + """Variant for B sequences. hint: "csa" | "hca" | "swa" (SWA-only layer) | None.""" + cfg = _plan_for(H, msq, max(1, B), epi, hint, cus, grouped) + if cfg.ws_need(B)[0] > MAX_GROUPS: + # more merge groups than protocol slots: run unsplit + cfg = dataclasses.replace( + cfg, + S=1, + ADAPT=0, + MINT=1, + SUB=(), + SPREAD=False, + SHORT=0, + SHORTM=False, + SHORTF=0, + OWN=False, + SK=0, + SKMEAN=False, + IDLE=(), + ) + return cfg + + +def _plan_for(H, msq, B, epi, hint, cus, grouped) -> V4Cfg: + if grouped: + return _plan_grouped(msq, B, epi, hint, cus) + if H == 16 and msq == 1: + return _plan_h16(B, epi, hint, cus) + if B >= _POLICY_Q16[(H, msq)]: + return _with_subs(_q16_plan(H, msq, B, epi, cus), B, cus) + for bmax, rb, s in _POLICY_WIDE[(H, msq)]: + if bmax is None or B <= bmax: + break + cfg = V4Cfg(H=H, MSQ=msq, RB=rb, S=s, EPI=epi, KC=True, SPREAD=True) + return _with_subs(cfg, B, cus) + + +def _plan_h16(B, epi, hint, cus) -> V4Cfg: + tpad = -(-B // 8) * 8 + adapt = {} + if hint == "swa": + s = 1 + elif epi == "bf16" and hint is None: + s = min(_H16_MAX_S, max(1, cus // tpad)) + if 16 <= tpad <= 2 * cus: + # ragged batches: the key-balanced grid (V4Cfg.SK; past cus sequences two + # CTAs per CU) + sk = cus * -(-tpad // cus) + cfg = _sk_plan(B, epi, cus, 16 * cus // sk, SK=sk, SKT=_H16_SKT[sk > cus]) + if 2 * tpad > cus and tpad <= cus * _PAST_HALF_K // 32: + cfg = dataclasses.replace(cfg, SKD=2, SKT=14, SKMEAN=True, SHORT=4) + return cfg + if tpad > cus: + s = 2 + adapt = {"ADAPT": cus, "MINT": 16} + elif tpad < 16: + short = V4Cfg(H=16, RB=1, EPI=epi, KC=True) + adapt = {"SUB": ((_H16_SHORT_KEYS, short),)} + else: + s = 1 + for tb, v in _POLICY_H16.items(): + if tpad <= tb: + s = v[0 if hint == "csa" else 1] + break + if hint == "hca" and tpad > 40: + # ragged HCA batches (a few long sessions set the longest stream): the + # key-balanced grid; past one sequence per CU, run-time split counts + if tpad > cus: + return V4Cfg(H=16, RB=1, S=8, EPI=epi, KC=True, ADAPT=cus, MINT=4) + return _sk_plan(B, epi, cus, 8, SK=cus, SKT=6) + return V4Cfg(H=16, RB=1, S=min(s, MAX_S), EPI=epi, KC=True, **adapt) + + +def _sk_plan(B, epi, cus, S, **kw) -> V4Cfg: + """H16 key-balanced grid (V4Cfg.SK) of at most max(S, SKD) splits, else SKD + splits per sequence: cus // T_pad, or cus // B on V4Cfg.TAIL where that is more""" + tpad = -(-B // 8) * 8 + skd = max(1, min(_H16_MAX_S, cus // tpad)) + tail = skd < cus // B <= _H16_TAIL_SKD + skd = cus // B if tail else skd + return V4Cfg( + H=16, + RB=1, + EPI=epi, + KC=True, + S=max(S, skd), + SKD=skd, + TAIL=tail, + MINT=3 if 4 * tpad > cus else 1, + SHORT=_H16_SHORT_KEYS // 64, + **kw, + ) + + +def _with_subs(cfg: V4Cfg, B: int, cus: int) -> V4Cfg: + """cfg + its stream-length bodies where they fit (at most the sequence's CTAs; the + short body at most one CTA per CU over the batch; under Q16 16-row K or 32-row Q16 + bodies).""" + if cfg.S == 1 or cfg.XR or cfg.IDLE: + return cfg + if cfg.Q16 and cfg.RB == 4 and B > 16 and cfg.S >= 3: + rb2 = dataclasses.replace(cfg, RB=2, S=1, SHORT=0, PF16=False) + keys = max(_Q16_RB2_KEYS // cfg.S, _Q16_RB2_MIN) + # (the main's short streams all take the body) + return dataclasses.replace(cfg, SHORT=0, SUB=((keys, rb2),)) + subs = [] + keys = next((k for bmax, k in _SHORT_KEYS if B <= bmax), 0) + if cfg.Q16 and cfg.RB == 2: + keys = min(keys, _Q16_SHORT_KEYS) + rbs = (1,) if cfg.Q16 else (1, 2) + for rb in rbs if keys else (): + nrg = cfg.H * cfg.MSQ // (16 * rb) + if nrg <= cfg.NRG * cfg.S and B * nrg <= cus: + subs.append((keys, _k_body(cfg, rb, 1))) + break + for bmax, bmin, mids in _POLICY_MID.get((cfg.H, cfg.MSQ), ()): + if bmin <= B <= bmax: + for rb, s, mkeys in mids: + sub = _k_body(cfg, rb, s) + fits = rb in rbs and sub.NRG * s <= cfg.NRG * cfg.S and sub != cfg + if fits and mkeys > keys: + subs.append((mkeys, sub)) + keys = mkeys + break + return dataclasses.replace(cfg, SUB=tuple(subs)) if subs else cfg + + +def _q16_plan(H: int, msq: int, B: int, epi: str, cus: int) -> V4Cfg: + rb = 4 if H * msq % 64 == 0 else 2 + nrg = H * msq // (16 * rb) + bpad = -(-B // 8) * 8 + s = max(1, min(MAX_S, cus // (bpad * nrg))) + xr = nrg > 1 and bpad <= 8 + if nrg > 1 and bpad > 8: + # row groups on consecutive XCDs where that allows more splits + s_xr = min(MAX_S, cus // 8 // -(-B * nrg // 8)) + if s_xr > s: + s, xr = s_xr, True + # the units past the last complete XCD octet over all XCDs (V4Cfg.TAIL) + tail_ok = rb == 4 and s < _TAIL_S and 8 < B < cus // 8 * nrg and B % 8 + s_tl = _tail_splits(B, nrg, xr, cus) if tail_ok else 0 + tail, s = s_tl > s, max(s, s_tl) + mean_short = rb == 4 and nrg == 1 and 2 * bpad == cus + past_half = ( + s == 1 + and not xr + and cus < 2 * bpad * nrg + and bpad * nrg <= cus * _PAST_HALF.get((rb, nrg), 0) // 32 + ) + if past_half: + s = 2 + if s == 1 and not xr and (bpad * nrg < cus or nrg > 1 and bpad <= 512): + s, idle = 2, (cus // 8, _IDLE_X2, _IDLE_F, _IDLE_W) + if bpad * nrg >= cus: + s, idle = 4, (bpad // 8 * nrg, _IDLE_XL, _IDLE_F, _IDLE_WL) + return V4Cfg( + H=H, + MSQ=msq, + RB=rb, + S=s, + EPI=epi, + LAYOUT="RD", + Q16=True, + PF16=True, + PFN=cus if bpad * nrg > cus else 0, + IDLE=idle, + ) + if s == 1: + short = 0 + elif mean_short: + short = 64 + elif past_half: + short = 1 << 16 + elif bpad <= 8: + short = 6 + else: + short = _Q16_SHORT_TILES.get(s, 8) + cfg = V4Cfg( + H=H, + MSQ=msq, + RB=rb, + S=s, + EPI=epi, + LAYOUT="RD", + Q16=True, + SHORT=short, + SHORTM=(mean_short or past_half) and s > 1, + SHORTF=16 if past_half else 0, + PF16=s > 1, + XR=xr, + PFN=cus if bpad * nrg > cus else 0, + TAIL=tail, + ) + if rb == 4 and nrg == 1 and bpad >= cus and epi == "bf16": + cfg = dataclasses.replace( + cfg, PV16R=(_PV16_LO, _PV16_HI[0] if bpad <= cus else _PV16_HI[1]) + ) + return cfg + + +def _tail_splits(B: int, nrg: int, xr: bool, cus: int) -> int: + # most splits under V4Cfg.TAIL with at most one CTA per CU on every XCD (one row + # group: 7/8 of them; a full XCD costs up to 1.25x at 1k-4k keys; XR: at most 12, + # more cost ~8 % at 200-500 keys) + units, per = (B * nrg, 1) if xr else (B, nrg) + cap = cus // 8 if nrg > 1 else cus * 7 // 64 + s = 12 if xr else MAX_S + while s > 1 and units // 8 * s * per + -(-(units % 8) * s * per // 8) > cap: + s -= 1 + return s + + +def _k_body(cfg: V4Cfg, rb: int, s: int) -> V4Cfg: + return V4Cfg( + H=cfg.H, + MSQ=cfg.MSQ, + RB=rb, + S=s, + EPI=cfg.EPI, + KC=True, + SPREAD=cfg.SPREAD, + TAIL=cfg.TAIL, + ) + + +def _plan_grouped(msq: int, B: int, epi: str, hint: str | None, cus: int) -> V4Cfg: + """Grouped verify at H = 16: B groups of up to msq drafts sharing one stream.""" + hca = hint in (None, "hca") + if epi == "bf16" and (B > 32 or (B == 32 and hca)): + return _grouped_q16(msq, B, hca, cus) + if B >= 32 and hca: + # long streams on one 128-row RQ CTA per (group, split), short ones on K + short = V4Cfg( + H=16, + MSQ=GROUPED_MAX_MSQ, + RB=2 if B >= 64 else 1, + EPI=epi, + KC=True, + MASK="bounds", + ) + return V4Cfg( + H=16, + MSQ=GROUPED_MAX_MSQ, + RB=GROUPED_MAX_MSQ, + S=4, + EPI=epi, + LAYOUT="RQ", + MASK="bounds", + OWN=True, + SUB=((32 * (_GROUPED_RQ_TILES - 1), short),), + ) + if B > 32: + return V4Cfg( + H=16, + MSQ=msq + (msq & 1), + RB=2, + EPI=epi, + KC=hint != "swa", + MASK="bounds", + ) + s = 1 if hint == "swa" or B > 16 else (2 if B > 8 else 4) + return V4Cfg(H=16, MSQ=msq, RB=1, S=s, EPI=epi, MASK="bounds") + + +def _grouped_q16(msq: int, B: int, hca: bool, cus: int) -> V4Cfg: + """32-row Q16 CTAs (two drafts each), 64-row ones where the 32-row grid exceeds + one CTA per CU and they need fewer.""" + mp = msq + (msq & 1) + rb = 2 + nrg = 16 * mp // 32 + bpad = -(-B // 8) * 8 + mp4 = -(-msq // 4) * 4 + if bpad * nrg > cus and 16 * mp4 // 64 < nrg: + mp, rb, nrg = mp4, 4, 16 * mp4 // 64 + s = max(1, min(4, cus // (bpad * nrg))) if hca else 1 + return V4Cfg( + H=16, + MSQ=mp, + RB=rb, + S=s, + LAYOUT="RD", + MASK="bounds", + Q16=True, + SHORT=16 if s > 1 else 0, + PF16=s > 1, + ) + + +class _Workspace: + """Grow-only merge scratch of one (device, stream): protocol slots (hdr, zero + between launches) + split partials (buf). A new stream takes a spare hdr zeroed + at an earlier eager launch, so a graph captured on it replays no memset. Old + buffers stay alive: a captured graph may still reference them.""" + + def __init__(self, device): + self.device = device + self.hdr = None + self.buf = None + self._keep = [] + + def ensure(self, nbytes: int): + if self.hdr is None: + self.hdr = _HDR_SPARE.pop(self.device.index, None) + if self.hdr is None: + self.hdr = torch.zeros( + WS_HDR // 4, dtype=torch.int32, device=self.device + ) + if ( + not torch.cuda.is_current_stream_capturing() + and self.device.index not in _HDR_SPARE + ): + _HDR_SPARE[self.device.index] = _zeroed_hdr(self.device) + need = -(-nbytes // 4) + if self.buf is None or self.buf.numel() < need: + if self.buf is not None: + self._keep.append(self.buf) + need = max(need, self.buf.numel() * 2) + self.buf = torch.empty(need, dtype=torch.int32, device=self.device) + return self + + +_WORKSPACES: dict = {} +_HDR_SPARE: dict = {} +_ZERO_STREAMS: dict = {} +_COMPILED: set = set() +_DUMMY: dict = {} + + +def _zeroed_hdr(device): + # zeroed on an idle side stream, before any stream can use it + zs = _ZERO_STREAMS.get(device.index) + if zs is None: + zs = _ZERO_STREAMS[device.index] = torch.cuda.Stream(device) + with torch.cuda.stream(zs): + hdr = torch.zeros(WS_HDR // 4, dtype=torch.int32, device=device) + zs.synchronize() + return hdr + + +def _workspace(device) -> _Workspace: + stream = torch.cuda.current_stream(device) + key = (device.index, stream.cuda_stream) + ws = _WORKSPACES.get(key) + if ws is None: + ws = _WORKSPACES[key] = _Workspace(device) + return ws + + +def _launch( + cfg: V4Cfg, + B, + q, + q_rope, + kv, + kv_rope, + qo_indptr, + kv_indptr, + kv_indices, + sink, + out, + out_s0, + out_s1, + xs=None, + positions=None, + freqs=None, + bounds=None, +): + device = q.device + dmy = _DUMMY.get(device.index) + if dmy is None: + dmy = _DUMMY[device.index] = torch.zeros(64, dtype=torch.int64, device=device) + groups, nbytes = cfg.ws_need(B) + if groups: + ws = _workspace(device).ensure(nbytes) + hdr, wsb = ws.hdr, ws.buf + else: + hdr = wsb = dmy + if cfg not in _COMPILED and torch.cuda.is_current_stream_capturing(): + raise RuntimeError( + f"flydsl_mla_v4_decode: variant {cfg} not compiled; run this batch size " + "once eagerly before graph capture" + ) + u8 = torch.uint8 + epi = cfg.EPI == "invrope_mxfp8" + _run_compiled( + build_mla_v4_decode(cfg), + ptr_arg(q.view(u8), fx.Uint8), + ptr_arg(q_rope, fx.BFloat16), + ptr_arg(kv.view(u8), fx.Uint8), + ptr_arg(kv_rope, fx.BFloat16), + ptr_arg(dmy if qo_indptr is None else qo_indptr, fx.Int32), + ptr_arg(kv_indptr, fx.Int32), + ptr_arg(kv_indices, fx.Int32), + ptr_arg(sink, fx.Float32), + ptr_arg(out.view(u8) if epi else out, fx.Uint8 if epi else fx.BFloat16), + ptr_arg(dmy if xs is None else xs.view(u8), fx.Uint8), + ptr_arg(dmy if positions is None else positions, fx.Int64), + ptr_arg(dmy if freqs is None else freqs, fx.Float32), + ptr_arg(wsb, fx.Int32), + ptr_arg(dmy if bounds is None else bounds, fx.Int32), + ptr_arg(hdr, fx.Int32), + B, + kv.shape[0], + out_s0, + out_s1, + fx.Stream(torch.cuda.current_stream(device)), + ) + _COMPILED.add(cfg) + + +def _check_bounds(q_kv_bounds, total_q): + if q_kv_bounds is not None: + assert q_kv_bounds.dtype == torch.int32 and q_kv_bounds.is_contiguous() + assert q_kv_bounds.dim() == 2 and q_kv_bounds.shape[1] == 4 + assert q_kv_bounds.shape[0] >= total_q, (q_kv_bounds.shape, total_q) + + +def flydsl_mla_decode_fwd_v4_nm( + q, + qrope, + kv_buffer, + kvrope, + output, + qo_indptr, + kv_indptr, + kv_page_indices, + max_seqlen_q, + *, + sink, + hint=None, + q_kv_bounds=None, +): + """aiter.mla.mla_decode_fwd_v4_nm on gfx950: writes the final bf16 attention into + ``output`` [total_q, H, 512] (rows without keys become NaN). + + hint: "csa" | "hca" | "swa" | None, only selects the split plan. + q_kv_bounds: optional int32 [total_q, 4] (a0, a1, b0, b1); q row r attends stream + position j of its sequence iff a0 <= j < a1 or b0 <= j < b1 (grouped verify). + """ + H = q.shape[1] + msq = int(max_seqlen_q) + grouped = q_kv_bounds is not None + if not flydsl_mla_v4_decode_supported(H, msq, grouped): + raise RuntimeError( + f"flydsl_mla_decode_fwd_v4_nm: no variant for num_heads={H}, " + f"max_seqlen_q={msq}, grouped={grouped}" + ) + B = qo_indptr.numel() - 1 + assert q.dim() == 3 and q.shape[2] == ROWB and q.element_size() == 1, q.shape + assert qrope.shape[1:] == (H, ROPE) and qrope.dtype == torch.bfloat16 + assert q.is_contiguous() and qrope.is_contiguous() + assert kv_buffer.element_size() == 1 and kvrope.dtype == torch.bfloat16 + kv = kv_buffer.reshape(-1, ROWB) + kv_rope = kvrope.reshape(-1, ROPE) + assert kv.shape[0] == kv_rope.shape[0] + assert sink.dtype == torch.float32 and sink.is_contiguous() and sink.numel() == H + for t in (qo_indptr, kv_indptr, kv_page_indices): + assert t.dtype == torch.int32 and t.is_contiguous(), t.dtype + assert output.dtype == torch.bfloat16 and output.shape[1:] == (H, DV) + assert output.stride(2) == 1 + _check_bounds(q_kv_bounds, q.shape[0]) + cfg = _plan(H, msq, B, "bf16", hint, _num_cus(q.device.index), grouped) + if B > 0: + _launch( + cfg, + B, + q, + qrope, + kv, + kv_rope, + qo_indptr, + kv_indptr, + kv_page_indices, + sink, + output, + output.stride(0), + output.stride(1), + bounds=q_kv_bounds, + ) + return output + + +def flydsl_mla_v4_decode_fused( + q, + q_rope, + kv, + kv_rope, + kv_indptr, + kv_indices, + sink, + positions, + freqs, + compress_ratio, + *, + qo_indptr=None, + max_seqlen_q: int = 1, + q_kv_bounds=None, +): + """DeepSeek-V4 decode tail in one launch: attention + split merge + inverse RoPE of + dims 448..511 + wo_a mxfp8 quant. + + q / q_rope / kv / kv_rope / sink: as flydsl_mla_decode_fwd_v4_nm; one sequence per + token (kv_indptr [T + 1]) unless q_kv_bounds + qo_indptr + max_seqlen_q give + grouped-verify streams. positions [T] int64; freqs [max_pos, 64] fp32 = + view_as_real(freqs_cis). compress_ratio: 4 CSA, 128 HCA, else SWA-only. + Returns (xq [T, H*512] fp8_e4m3fn, xs [T, H*4] e8m0); tokens without keys get NaN. + """ + T, H = q.shape[0], q.shape[1] + msq = int(max_seqlen_q) + grouped = q_kv_bounds is not None + if not (msq == 1 or grouped) or not flydsl_mla_v4_decode_supported(H, msq, grouped): + raise RuntimeError( + f"flydsl_mla_v4_decode_fused: no variant for num_heads={H}, " + f"max_seqlen_q={msq}, grouped={grouped}" + ) + if grouped: + assert qo_indptr is not None and qo_indptr.dtype == torch.int32 + B = qo_indptr.numel() - 1 + else: + B = T + assert q.shape[2] == ROWB and q_rope.shape[1:] == (H, ROPE) + assert q.is_contiguous() and q_rope.is_contiguous() and q.element_size() == 1 + assert q_rope.dtype == kv_rope.dtype == torch.bfloat16 + assert kv_indptr.dtype == kv_indices.dtype == torch.int32 + assert sink.dtype == freqs.dtype == torch.float32 and positions.dtype == torch.int64 + assert sink.is_contiguous() and freqs.is_contiguous() and kv.element_size() == 1 + assert freqs.shape[-1] == ROPE + kv = kv.reshape(-1, ROWB) + kv_rope = kv_rope.reshape(-1, ROPE) + assert kv.shape[0] == kv_rope.shape[0] + xq = torch.empty((T, H * DV), dtype=torch.float8_e4m3fn, device=q.device) + xs = torch.empty((T, H * DV // 128), dtype=torch.uint8, device=q.device) + _check_bounds(q_kv_bounds, T) + hint = {4: "csa", 128: "hca"}.get(compress_ratio, "swa") + cfg = _plan(H, msq, B, "invrope_mxfp8", hint, _num_cus(q.device.index), grouped) + if T > 0 and B > 0: + _launch( + cfg, + B, + q, + q_rope, + kv, + kv_rope, + None if cfg.MSQ == 1 else qo_indptr, + kv_indptr, + kv_indices, + sink, + xq, + 0, + 0, + xs, + positions, + freqs, + q_kv_bounds, + ) + return xq, xs diff --git a/python/sglang/kernels/ops/attention/dsv4/flydsl_v4nm/mla_v4_grouped.py b/python/sglang/kernels/ops/attention/dsv4/flydsl_v4nm/mla_v4_grouped.py new file mode 100644 index 0000000000..827109b9be --- /dev/null +++ b/python/sglang/kernels/ops/attention/dsv4/flydsl_v4nm/mla_v4_grouped.py @@ -0,0 +1,174 @@ +# SPDX-License-Identifier: MIT +# Copyright (C) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. +# +# Vendored from ROCm/aiter#5882 (https://github.com/ROCm/aiter/pull/5882), +# commit 03359c43df (03359c43df40abf91ac38a2040c9b8e284d92297), +# file aiter/ops/flydsl/mla_v4_grouped.py. +# Unchanged except for import paths and sglang formatting (isort, ruff format). +# TODO(aiter#5882): remove once ROCm/aiter#5882 is in the aiter pin + +"""Grouped DSpark verify streams for the FlyDSL v4 decode. + +The num_draft tokens of a request (positions p-nd+1 .. p) share one stream at an HCA / +SWA-only layer: [compressed tail of the last draft (C entries)][SWA ring slots of +positions p-n+1 .. p], n = min(p+1, win+nd-1) (the sglang#41120 layout). q_kv_bounds +gives each draft its exact key set in it: [0, C_q) and its own window. +""" + +from __future__ import annotations + +import torch +import triton +import triton.language as tl + +__all__ = ["flydsl_mla_v4_grouped_verify_meta"] + +GROUPED_MAX_MSQ = 8 # drafts per request + + +@triton.jit +def _grouped_meta_kernel( + idx_ptr, + kv_indptr_ptr, + qo_indptr_ptr, + bnd_ptr, + slot_ptr, + pos_ptr, + tail_len_ptr, + tail_page_ptr, + G, + ring_stride, + swa_pages, + tail_stride, + ND: tl.constexpr, + WIN: tl.constexpr, + TAIL_B: tl.constexpr, + WIN_B: tl.constexpr, + GB: tl.constexpr, + HAS_TAIL: tl.constexpr, + MAXND: tl.constexpr, +): + g = tl.program_id(0) + # start of this group's stream: the previous groups' lengths + base = 0 + for off in tl.range(0, G, GB): + gg = off + tl.arange(0, GB) + m = gg < g + last = gg * ND + ND - 1 + p = tl.load(pos_ptr + last, mask=m, other=0).to(tl.int32) + if HAS_TAIL: + c = tl.load(tail_len_ptr + last, mask=m, other=0).to(tl.int32) + else: + c = tl.zeros([GB], tl.int32) + n = tl.minimum(p + 1, WIN + ND - 1) + base += tl.sum(tl.where(m, c + n, 0)) + first = g * ND + last = first + ND - 1 + p = tl.load(pos_ptr + last).to(tl.int32) + if HAS_TAIL: + c = tl.load(tail_len_ptr + last).to(tl.int32) + else: + c = 0 + n = tl.minimum(p + 1, WIN + ND - 1) + tl.store(kv_indptr_ptr + g + 1, base + c + n) + tl.store(qo_indptr_ptr + g + 1, first + ND) + if g == 0: + tl.store(kv_indptr_ptr, 0) + tl.store(qo_indptr_ptr, 0) + if HAS_TAIL: + for off in tl.range(0, c, TAIL_B): + jt = off + tl.arange(0, TAIL_B) + mt = jt < c + pi = tl.load( + tail_page_ptr + last.to(tl.int64) * tail_stride + jt, mask=mt, other=-1 + ) + tl.store( + idx_ptr + base + jt, tl.where(pi >= 0, pi + swa_pages, -1), mask=mt + ) + slot = tl.load(slot_ptr + last).to(tl.int64) + i = tl.arange(0, WIN_B) + ap = (p - n + 1 + i).to(tl.int64) + tl.store( + idx_ptr + base + c + i, + (slot * ring_stride + ap % ring_stride).to(tl.int32), + mask=i < n, + ) + # per-row bounds, stream relative + q = tl.arange(0, MAXND) + mq = q < ND + t = first + q + pq = tl.load(pos_ptr + t, mask=mq, other=0).to(tl.int32) + if HAS_TAIL: + cq = tl.load(tail_len_ptr + t, mask=mq, other=0).to(tl.int32) + else: + cq = tl.zeros([MAXND], tl.int32) + s0 = p - n + 1 + tb = bnd_ptr + t.to(tl.int64) * 4 + tl.store(tb + 0, tl.zeros([MAXND], tl.int32), mask=mq) + tl.store(tb + 1, cq, mask=mq) + tl.store(tb + 2, c + tl.maximum(0, pq - WIN + 1 - s0), mask=mq) + tl.store(tb + 3, c + pq - s0 + 1, mask=mq) + + +def flydsl_mla_v4_grouped_verify_meta( + state_slot: torch.Tensor, + positions: torch.Tensor, + tail_len: torch.Tensor | None, + tail_page_indices: torch.Tensor | None, + *, + win: int, + ring_stride: int, + swa_pages: int, + num_draft: int, +): + """(kv_indices, kv_indptr, qo_indptr, q_kv_bounds) of the grouped streams, one + graph-capturable launch; N = requests * num_draft tokens, request-major. + + state_slot / positions [N]; tail_len [N] / tail_page_indices [N, Wc] int32: the + committed compressed entries per token (None: SWA-only). kv_indices is sized for + the worst case. Use with max_seqlen_q = num_draft. + """ + assert 1 <= num_draft <= GROUPED_MAX_MSQ + dev = positions.device + N = positions.shape[0] + assert N % num_draft == 0, (N, num_draft) + G = N // num_draft + has_tail = tail_len is not None and tail_page_indices is not None + Wc = tail_page_indices.shape[1] if has_tail else 0 + idx = torch.empty( + max(G * (Wc + win + num_draft - 1), 1), dtype=torch.int32, device=dev + ) + kv_indptr = torch.empty(G + 1, dtype=torch.int32, device=dev) + qo_indptr = torch.empty(G + 1, dtype=torch.int32, device=dev) + bnd = torch.empty((N, 4), dtype=torch.int32, device=dev) + if G == 0: + kv_indptr.zero_() + qo_indptr.zero_() + return idx, kv_indptr, qo_indptr, bnd + assert state_slot.is_contiguous() and positions.is_contiguous() + if has_tail: + assert tail_len.is_contiguous() and tail_page_indices.stride(1) == 1 + tp = tail_page_indices if has_tail else positions + _grouped_meta_kernel[(G,)]( + idx, + kv_indptr, + qo_indptr, + bnd, + state_slot, + positions, + tail_len if has_tail else positions, + tp, + G, + ring_stride, + swa_pages, + tp.stride(0) if has_tail else 0, + ND=num_draft, + WIN=win, + TAIL_B=min(1024, triton.next_power_of_2(max(Wc, 1))), + WIN_B=triton.next_power_of_2(win + num_draft - 1), + GB=min(1024, triton.next_power_of_2(G)), + HAS_TAIL=has_tail, + MAXND=GROUPED_MAX_MSQ, + num_warps=4, + ) + return idx, kv_indptr, qo_indptr, bnd diff --git a/python/sglang/kernels/ops/attention/dsv4/unified_kv_kernels/runtime.py b/python/sglang/kernels/ops/attention/dsv4/unified_kv_kernels/runtime.py index 5b60c02202..53e12a5a00 100644 --- a/python/sglang/kernels/ops/attention/dsv4/unified_kv_kernels/runtime.py +++ b/python/sglang/kernels/ops/attention/dsv4/unified_kv_kernels/runtime.py @@ -27,6 +27,7 @@ on via ``topk_length``); the per-token compressed count is recovered from the from __future__ import annotations +import logging from functools import lru_cache from typing import Optional, Tuple @@ -49,6 +50,8 @@ from sglang.kernels.ops.attention.dsv4.unified_kv_kernels.paged_prefill import ( sparse_attn_v4_paged_prefill, ) +logger = logging.getLogger(__name__) + # --------------------------------------------------------------------------- # SWA ring scatter @@ -282,6 +285,289 @@ def _decode_fp8_tail_splits(T: int, compress_ratio: Optional[int]) -> int: return max(1, min(_DECODE_SPLIT_TAIL_VALUE, _cu_count() // T)) +# --------------------------------------------------------------------------- +# Split plan from aiter's v4 nm latency model (ROCm/aiter#6126, ATOM#2462) +# --------------------------------------------------------------------------- +# t(i) ~= L + W(i) * (C + A * kv_len / i) + [i > 1] * (S + D * num_seqs * i), +# W(i) = ceil(num_seqs * ceil(H / 64) * i / CU). The gfx950 fit is aiter's +# (gqa=128, qSeqLen=1, graph replay, uniform KV). Used only when the installed +# aiter predates get_mla_v4_nm_num_kv_splits. +_V4_NM_SPLIT_COST = {"gfx950": (12.74, 3.21, 0.0300, 3.38, 0.076)} +V4_NM_MAX_SPLITS = 16 +# rows the constant split table covers; the asm wraps byte offsets past 32768 +V4_NM_SPLIT_TABLE_ROWS = 32768 + + +def _ported_num_kv_splits( + num_seqs: int, num_heads: int, kv_len: int, gfx: str, cu_num: int +) -> Optional[int]: + cost = _V4_NM_SPLIT_COST.get(gfx) + if cost is None: + return None + L, C, A, S, D = cost + tg_factor = max(1, -(-num_heads // 64)) + kv_len = max(1, kv_len) + best_t, best_s = None, 1 + for s in range(1, V4_NM_MAX_SPLITS + 1): + waves = -(-num_seqs * tg_factor * s // cu_num) + t = L + waves * (C + A * kv_len / s) + if s > 1: + t += S + D * num_seqs * s + if best_t is None or t < best_t: + best_t, best_s = t, s + return best_s + + +@lru_cache(maxsize=None) +def fp8_decode_num_kv_splits( + num_seqs: int, num_heads: int, kv_len: int +) -> Optional[int]: + """aiter's latency-model split count for ``mla_decode_fwd_v4_nm``, or None + (no model for this arch: keep the existing pick). Host only, cached: the + inputs are fixed per captured graph, so capture and replay agree.""" + try: + from aiter.mla import get_mla_v4_nm_num_kv_splits + except ImportError: + get_mla_v4_nm_num_kv_splits = None + if get_mla_v4_nm_num_kv_splits is not None: + return int(get_mla_v4_nm_num_kv_splits(num_seqs, num_heads, kv_len)) + from aiter.jit.utils.chip_info import get_gfx + + return _ported_num_kv_splits(num_seqs, num_heads, kv_len, get_gfx(), _cu_count()) + + +_split_tables: dict = {} + + +def fp8_decode_split_table(device: torch.device) -> Optional[torch.Tensor]: + """``[V4_NM_MAX_SPLITS, V4_NM_SPLIT_TABLE_ROWS + 1]`` int32, row ``s - 1`` + the uniform ``split_indptr`` ``[0, s, 2s, ...]``. Built once per device + outside capture and never written again, so every plan is a slice of + constant memory (no per-layer arange). None if it would be built under + capture: that graph keeps the wrapper's own pick.""" + device = torch.device(device) + key = device.index if device.index is not None else torch.cuda.current_device() + table = _split_tables.get(key) + if table is None: + if torch.cuda.is_current_stream_capturing(): + return None + splits = torch.arange(1, V4_NM_MAX_SPLITS + 1, dtype=torch.int32, device=device) + table = splits[:, None] * torch.arange( + V4_NM_SPLIT_TABLE_ROWS + 1, dtype=torch.int32, device=device + ) + _split_tables[key] = table + return table + + +def fp8_decode_split_plan( + n_seq: int, num_heads: int, kv_len_hint: int, device: torch.device +) -> Optional[Tuple[int, torch.Tensor]]: + """(num_kv_splits, split_indptr) from the latency model, or None.""" + if n_seq > V4_NM_SPLIT_TABLE_ROWS: + return None + s = fp8_decode_num_kv_splits(n_seq, num_heads, int(kv_len_hint)) + if s is None: + return None + table = fp8_decode_split_table(device) + if table is None: + return None + return s, table[s - 1, : n_seq + 1] + + +# --------------------------------------------------------------------------- +# HCA persistent decode (aiter mla_decode_fwd_v4_nm_ps, ROCm/aiter#6126) +# --------------------------------------------------------------------------- +HCA_PS_HEADS = 128 +HCA_PS_MAX_ROWS = 32768 +HCA_PS_PARTITIONS = 128 +_hca_ps_workspaces: dict = {} + + +def _hca_ps_api(): + try: + from aiter.mla import get_mla_v4_nm_ps_workspace, mla_decode_fwd_v4_nm_ps + except ImportError: + return None + return get_mla_v4_nm_ps_workspace, mla_decode_fwd_v4_nm_ps + + +def _dev_key(device) -> int: + device = torch.device(device) + return device.index if device.index is not None else torch.cuda.current_device() + + +def prepare_fp8_hca_persist(device, num_heads: int) -> bool: + """Allocate this device's persistent-decode workspace (~64 MiB) when the + switch is on and the kernel can serve this rank. Call outside capture, + before warmup; idempotent. One workspace per device is enough because + every decode layer runs on the forward's compute stream in order.""" + from sglang.srt.environ import envs + from sglang.srt.utils import is_gfx95_supported + + if not envs.SGLANG_DSV4_FP8_HCA_PERSIST.get() or num_heads != HCA_PS_HEADS: + return False + if not is_gfx95_supported(): + return False + key = _dev_key(device) + if key in _hca_ps_workspaces: + return True + api = _hca_ps_api() + if api is None or torch.cuda.is_current_stream_capturing(): + return False + _hca_ps_workspaces[key] = api[0]( + torch.device("cuda", key), num_partitions=HCA_PS_PARTITIONS + ) + return True + + +def fp8_hca_persist_wanted( + *, + T: int, + H: int, + v_head_dim: int, + compress_ratio: Optional[int], + max_seqlen_q: int, + per_token: bool, + device, +) -> bool: + from sglang.srt.environ import envs + + if not ( + compress_ratio == 128 + and H == HCA_PS_HEADS + and v_head_dim == 512 + and max_seqlen_q == 1 + and per_token + and envs.SGLANG_DSV4_FP8_HCA_PERSIST.get() + and envs.SGLANG_DSV4_FP8_HCA_PERSIST_MIN_ROWS.get() <= T <= HCA_PS_MAX_ROWS + ): + return False + # eager calls may allocate it late; a graph captured without one keeps asm + return prepare_fp8_hca_persist(device, H) + + +def decode_fp8_hca_ps( + *, + q: torch.Tensor, # [T, 128, 512] fp8 packed + q_rope: torch.Tensor, # [T, 128, 64] bf16 + unified_kv: torch.Tensor, + unified_kv_rope: torch.Tensor, + kv_indices: torch.Tensor, + kv_indptr: torch.Tensor, + attn_sink: torch.Tensor, +) -> torch.Tensor: + """One-launch HCA decode: the kernel plans the split from ``kv_indptr`` on + the GPU and merges the partials itself. Rows with an empty KV range are + left unwritten (the builders never emit one; see decode_fp8_2buff).""" + T = q.shape[0] + out = q_rope.new_empty((T, HCA_PS_HEADS, 512)) + if T == 0: + return out + _hca_ps_api()[1]( + q, + q_rope, + unified_kv, + unified_kv_rope, + kv_indptr[: T + 1], + kv_indices, + attn_sink, + _hca_ps_workspaces[_dev_key(q.device)], + out=out, + ) + return out + + +# TODO(aiter#5882): remove once ROCm/aiter#5882 is in the aiter pin +# (with it: the in-tree fallback below, flydsl_v4nm/ and +# SGLANG_DSV4_FLYDSL_DECODE_SOURCE; aiter's copy is then the only one). +_FLYDSL_V4_DECODE_SOURCES = ("auto", "aiter", "intree") + + +@lru_cache(maxsize=1) +def _flydsl_v4_decode(): + """The FlyDSL v4 nm decode API (ROCm/aiter#5882) or None (asm kernel). + + SGLANG_DSV4_FLYDSL_DECODE=0 forces None. SGLANG_DSV4_FLYDSL_DECODE_SOURCE: + auto (default) takes aiter.ops.flydsl when the installed aiter ships it, else + sglang's in-tree copy (flydsl_v4nm); aiter / intree take only that one. + """ + from sglang.srt.environ import envs + + if not envs.SGLANG_DSV4_FLYDSL_DECODE.get(): + return None + source = envs.SGLANG_DSV4_FLYDSL_DECODE_SOURCE.get() + if source not in _FLYDSL_V4_DECODE_SOURCES: + raise ValueError( + f"SGLANG_DSV4_FLYDSL_DECODE_SOURCE={source!r}, expected one of " + f"{_FLYDSL_V4_DECODE_SOURCES}" + ) + fly = None + if source != "intree": + try: + from aiter.ops import flydsl as fly + + fly.flydsl_mla_v4_decode_supported # resolves the lazy import + origin = "aiter.ops.flydsl" + except (ImportError, AttributeError): + fly = None + if fly is None and source != "aiter": + try: + from sglang.kernels.ops.attention.dsv4 import flydsl_v4nm as fly + + origin = "sglang in-tree copy of ROCm/aiter#5882" + except ImportError as e: # no flydsl, or an aiter before ROCm/aiter#6377 + fly = None + logger.warning("in-tree FlyDSL v4 nm decode unavailable: %s", e) + if fly is None: + logger.info("DSv4 fp8 decode: no FlyDSL v4 nm decode (source=%s)", source) + else: + logger.info("DSv4 fp8 decode: FlyDSL v4 nm decode from %s", origin) + return fly + + +def flydsl_decode_supported( + num_heads: int, max_seqlen_q: int = 1, grouped: bool = False +) -> bool: + fly = _flydsl_v4_decode() + if fly is None: + return False + if num_heads == 128: + from sglang.srt.environ import envs + + if not envs.SGLANG_DSV4_FLYDSL_DECODE_H128.get(): + return False + return fly.flydsl_mla_v4_decode_supported(num_heads, max_seqlen_q, grouped) + + +def _stream_hint(compress_ratio: Optional[int]) -> Optional[str]: + # FlyDSL split plans are keyed by stream kind + if compress_ratio is None: + return None + return {0: "swa", 4: "csa", 128: "hca"}[compress_ratio] + + +def flydsl_grouped_verify_pays( + num_heads: int, num_draft: int, num_reqs: int, compress_ratio: int, epi: str +) -> bool: + """aiter's measured policy: whether target verify should read one shared + stream per request on this layer kind, for output mode ``epi``.""" + return flydsl_decode_supported( + num_heads, num_draft, grouped=True + ) and _flydsl_v4_decode().flydsl_mla_v4_grouped_pays( + num_heads, num_draft, num_reqs, hint=_stream_hint(compress_ratio), epi=epi + ) + + +def build_grouped_verify_streams( + state_slot: torch.Tensor, positions: torch.Tensor, *args, **kwargs +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """(kv_indices, kv_indptr, qo_indptr, q_kv_bounds): one stream per request, + with per-draft key ranges equal to each draft's per-token stream.""" + return _flydsl_v4_decode().flydsl_mla_v4_grouped_verify_meta( + state_slot.contiguous(), positions.contiguous(), *args, **kwargs + ) + + def decode_fp8_2buff( *, q: torch.Tensor, # [T, H, nope_row_bytes] fp8 packed nope + inline e8m0 scale @@ -296,8 +582,17 @@ def decode_fp8_2buff( max_seqlen_q: int = 1, num_kv_splits: Optional[int] = None, compress_ratio: Optional[int] = None, + q_kv_bounds: Optional[torch.Tensor] = None, + kv_len_hint: Optional[int] = None, ) -> torch.Tensor: - """Decode over the two-pool fp8 unified_kv, through aiter's v4 nm asm kernel. + """Decode over the two-pool fp8 unified_kv, through aiter's FlyDSL v4 nm + decode (split merge in the same launch) where available, else the asm kernel. + + On the asm kernel, with ``SGLANG_DSV4_FP8_HCA_PERSIST`` the HCA stream + (H128, one q row per sequence) takes the persistent kernel, and with + ``SGLANG_DSV4_FP8_DECODE_SPLIT_MODEL`` plus ``kv_len_hint`` (the longest + KV a row of this stream can see, fixed per server) the split count comes + from aiter's latency model with a constant ``split_indptr``. Q arrives in the same packed form as the pool rows (nope fp8 + duplicated e8m0 tile scales) with its rope half beside it in bf16, which is why this @@ -314,6 +609,9 @@ def decode_fp8_2buff( ``max_seqlen_q`` above 1 puts several q rows in one tile, which the v4 dispatcher serves for gqa=16 at 1, 2 and 4 off the same 64-row tile. + + ``q_kv_bounds`` (FlyDSL only): the grouped verify streams of + ``build_grouped_verify_streams``. """ from aiter.mla import mla_decode_fwd_v4_nm @@ -350,15 +648,52 @@ def decode_fp8_2buff( rows = unified_kv.shape[0] out = q_rope.new_empty((n_seq * max_seqlen_q, H, v_head_dim)) - # Left None, the wrapper's occupancy heuristic picks it, folds the cross-split - # merge back into `out`, and leaves the final bf16 there whether or not it - # split. Pinning it to 1 costs 6.9x at bs=1 kv=2048. - # Sequences, not tokens: the wave-fitting rule counts the workgroups the - # kernel launches, which is n_seq x splits. They are the same number on the - # per-token path and differ once q rows share a tile. - if num_kv_splits is None and n_seq > _DECODE_SPLIT_TAIL_MIN_TOKENS: - num_kv_splits = _decode_fp8_tail_splits(n_seq, compress_ratio) - mla_decode_fwd_v4_nm( + # FlyDSL is a drop-in for the asm kernel: its own capture-safe split plan, + # split merge in the same launch + if flydsl_decode_supported(H, max_seqlen_q, q_kv_bounds is not None): + decode_fwd = _flydsl_v4_decode().flydsl_mla_decode_fwd_v4_nm + kwargs = dict(hint=_stream_hint(compress_ratio), q_kv_bounds=q_kv_bounds) + else: + assert q_kv_bounds is None, "grouped verify streams need FlyDSL" + if fp8_hca_persist_wanted( + T=T, + H=H, + v_head_dim=v_head_dim, + compress_ratio=compress_ratio, + max_seqlen_q=max_seqlen_q, + per_token=n_seq == T, + device=q.device, + ): + return decode_fp8_hca_ps( + q=q, + q_rope=q_rope, + unified_kv=unified_kv, + unified_kv_rope=unified_kv_rope, + kv_indices=kv_indices, + kv_indptr=kv_indptr, + attn_sink=attn_sink, + ) + split_indptr = None + if num_kv_splits is None and kv_len_hint is not None: + from sglang.srt.environ import envs + + if envs.SGLANG_DSV4_FP8_DECODE_SPLIT_MODEL.get(): + plan = fp8_decode_split_plan(n_seq, H, kv_len_hint, q.device) + if plan is not None: + num_kv_splits, split_indptr = plan + # Left None, the wrapper's occupancy heuristic picks it, folds the + # cross-split merge back into `out`, and leaves the final bf16 there + # whether or not it split. Pinning it to 1 costs 6.9x at bs=1 kv=2048. + # Sequences, not tokens: the wave-fitting rule counts the workgroups the + # kernel launches, which is n_seq x splits. They are the same number on + # the per-token path and differ once q rows share a tile. + if num_kv_splits is None and n_seq > _DECODE_SPLIT_TAIL_MIN_TOKENS: + num_kv_splits = _decode_fp8_tail_splits(n_seq, compress_ratio) + decode_fwd, kwargs = ( + mla_decode_fwd_v4_nm, + dict(num_kv_splits=num_kv_splits, split_indptr=split_indptr), + ) + decode_fwd( q, q_rope, unified_kv.view(rows, 1, 1, row_bytes), @@ -369,7 +704,7 @@ def decode_fp8_2buff( kv_indices, max_seqlen_q, sink=attn_sink, - num_kv_splits=num_kv_splits, + **kwargs, ) # No empty-segment mask: a CG-padded row gets seq_len 1 on the ring slot # ReqToTokenPool reserves, so the builders can't emit a zero-length one, and @@ -378,6 +713,41 @@ def decode_fp8_2buff( return out[:T] +def decode_fp8_2buff_wo_a_mxfp8( + *, + q: torch.Tensor, # [T, H, nope_row_bytes] fp8 packed nope + inline e8m0 scale + q_rope: torch.Tensor, # [T, H, rope_dim] bf16 + unified_kv: torch.Tensor, # [rows, nope_row_bytes] fp8 + unified_kv_rope: torch.Tensor, # [rows, rope_dim] bf16 + kv_indices: torch.Tensor, + kv_indptr: torch.Tensor, + attn_sink: torch.Tensor, # [H] fp32 + positions: torch.Tensor, # [T] int64 + rope_freqs: torch.Tensor, # [max_pos, rope_dim] fp32, view_as_real(freqs_cis) + compress_ratio: int, + qo_indptr: Optional[torch.Tensor] = None, + max_seqlen_q: int = 1, + q_kv_bounds: Optional[torch.Tensor] = None, +) -> Tuple[torch.Tensor, torch.Tensor]: + """``decode_fp8_2buff`` + inverse RoPE + wo_a mxfp8 quant in one FlyDSL + launch. Returns the wo_a operands (fp8 [T, H*512], e8m0 [T, H*4]).""" + return _flydsl_v4_decode().flydsl_mla_v4_decode_fused( + q, + q_rope, + unified_kv, + unified_kv_rope, + kv_indptr, + kv_indices, + attn_sink, + positions, + rope_freqs, + compress_ratio, + qo_indptr=qo_indptr, + max_seqlen_q=max_seqlen_q, + q_kv_bounds=q_kv_bounds, + ) + + @triton.jit def _fill_compress_tail_kernel( indices_ptr, # [*] int32 (out) diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index 55d7e22c66..ae45e1c3a5 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -1613,6 +1613,31 @@ class Envs: # only way to ask; on separate-KV it is the reverse -- --kv-cache-dtype # picks the buffer dtype and this switch is inert. SGLANG_DSV4_UNIFIED_KV_FP8 = EnvBool(False) + # fp8 unified_kv decode/verify on aiter's FlyDSL v4 nm decode when the + # installed aiter ships it; 0 keeps the asm kernel. + SGLANG_DSV4_FLYDSL_DECODE = EnvBool(True) + # ... which copy: "auto" = aiter's when the installed aiter ships it, else + # sglang's in-tree copy; "aiter" = aiter's only (asm without it); "intree" = + # the in-tree copy only. + # TODO(aiter#5882): remove once ROCm/aiter#5882 is in the aiter pin + SGLANG_DSV4_FLYDSL_DECODE_SOURCE = EnvStr("auto") + # ... also with 128 local heads (DP attention). Off: there the asm path + # below (split plan + HCA persistent, ROCm/aiter#6126) measured the same + # DP8 DSpark decode step on MI355X without depending on FlyDSL. + SGLANG_DSV4_FLYDSL_DECODE_H128 = EnvBool(False) + # fp8 unified_kv asm decode (when FlyDSL is off or absent): pick the KV + # split count from aiter's v4 nm latency model (ROCm/aiter#6126; ported + # when aiter predates it) with a constant split_indptr, per stream kind. + SGLANG_DSV4_FP8_DECODE_SPLIT_MODEL = EnvBool(True) + # context the HCA split plan is sized for, capped at the server context + # length (0: the server context length). Sizing for 1M over-splits HCA at + # AgentX lengths (32k-200k). + SGLANG_DSV4_FP8_DECODE_SPLIT_CTX = EnvInt(200_000) + # fp8 unified_kv asm decode: HCA layers with 128 local heads (DP attention) + # take aiter's persistent mla_decode_fwd_v4_nm_ps (in-kernel split + merge) + # for MIN_ROWS..32768 rows; needs aiter >= ROCm/aiter#6126. + SGLANG_DSV4_FP8_HCA_PERSIST = EnvBool(True) + SGLANG_DSV4_FP8_HCA_PERSIST_MIN_ROWS = EnvInt(15) # DeepSeek-V4.1 engram host table: keep the tables in host memory (layout # below) and gather rows from the GPU instead of sharding them over HBM. diff --git a/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py b/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py index b2badd8ade..98593b84c0 100644 --- a/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py +++ b/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py @@ -270,6 +270,16 @@ class UnifiedKvMetadata: gasm_kv_indptr: Optional[torch.Tensor] = None gasm_qo_indptr: Optional[torch.Tensor] = None + # Grouped target-verify streams for the FlyDSL fp8 decode: one per request + # on HCA / SWA-only layers, with per-draft key ranges (q_kv_bounds). + grp_qo_indptr: Optional[torch.Tensor] = None + grp_hca_indices: Optional[torch.Tensor] = None + grp_hca_indptr: Optional[torch.Tensor] = None + grp_hca_bounds: Optional[torch.Tensor] = None + grp_swa_indices: Optional[torch.Tensor] = None + grp_swa_indptr: Optional[torch.Tensor] = None + grp_swa_bounds: Optional[torch.Tensor] = None + # prefill/extend per-token mapping pf_state_slot: Optional[torch.Tensor] = None pf_chunk_start: Optional[torch.Tensor] = None @@ -298,6 +308,13 @@ class UnifiedKvMetadata: "gasm_indices", "gasm_kv_indptr", "gasm_qo_indptr", + "grp_qo_indptr", + "grp_hca_indices", + "grp_hca_indptr", + "grp_hca_bounds", + "grp_swa_indices", + "grp_swa_indptr", + "grp_swa_bounds", "csa_indices", "csa_indptr", "pf_state_slot", @@ -330,6 +347,13 @@ class UnifiedKvMetadata: "gasm_indices", "gasm_kv_indptr", "gasm_qo_indptr", + "grp_qo_indptr", + "grp_hca_indices", + "grp_hca_indptr", + "grp_hca_bounds", + "grp_swa_indices", + "grp_swa_indptr", + "grp_swa_bounds", "csa_indices", "csa_indptr", "pf_state_slot", @@ -971,6 +995,17 @@ class DeepseekV4HipRadixBackend( # DSpark draft workers verify gamma rows; the server arg keeps the # CUDA-side convention gamma + 1. self.target_verify_num_draft_tokens = self.speculative_num_draft_tokens - 1 + self._fp8_decode_num_heads = ( + model_runner.model_config.num_attention_heads // get_parallel().attn_tp_size + ) + # output mode of the fp8 verify: deepseek_v4 fuses the fp8 wo_a quant into it + self._fp8_verify_epi = ( + "invrope_mxfp8" + if any(getattr(m, "wo_a_fp8", False) for m in model_runner.model.modules()) + else "bf16" + ) + self._fp8_split_kv_hint: Dict[int, int] = {} + self._init_fp8_asm_decode_plan() # Past MAX_FUSED_ROWS the fp4 schedule falls back to AITER's preamble, # which frees the scratch its kernels read -- not capture-safe. self._fp4_graph_row_limit: Optional[int] = None @@ -2409,6 +2444,37 @@ class DeepseekV4HipRadixBackend( if current_raw is not None: self.forward_metadata = current_raw + def _init_fp8_asm_decode_plan(self) -> None: + """fp8 unified_kv asm decode: per-stream KV bound for the split plan + and, at init (outside capture, before warmup), the constant split + table and the HCA persistent workspace.""" + from sglang.kernels.ops.attention.dsv4.unified_kv_kernels.env_gate import ( + is_unified_kv_fp8, + ) + + if self.is_draft_worker or not is_unified_kv_fp8(): + return + from sglang.kernels.ops.attention.dsv4.unified_kv_kernels import runtime + + win = self.token_to_kv_pool.unified_swa_window + ctx = min( + envs.SGLANG_DSV4_FP8_DECODE_SPLIT_CTX.get() or self.max_context_len, + self.max_context_len, + ) + # longest KV one decode row of each stream can see + self._fp8_split_kv_hint = { + 0: win, + 4: win + self.index_topk, + 128: win + ctx // 128, + } + if envs.SGLANG_DSV4_FP8_DECODE_SPLIT_MODEL.get(): + runtime.fp8_decode_split_table(self.device) + if runtime.prepare_fp8_hca_persist(self.device, self._fp8_decode_num_heads): + logger.info( + "DSV4 fp8 decode: HCA layers on the persistent asm kernel " + f"(rows >= {envs.SGLANG_DSV4_FP8_HCA_PERSIST_MIN_ROWS.get()})" + ) + def _attach_unified_kv_decode_streams( self, core: DSV4AttnMetadata, @@ -2418,6 +2484,7 @@ class DeepseekV4HipRadixBackend( # state_slot maps each query token to its request slot; # target-verify repeats request slots for the draft tokens. from sglang.kernels.ops.attention.dsv4.unified_kv_kernels.env_gate import ( + is_unified_kv_fp8, is_unified_kv_triton, ) @@ -2493,6 +2560,43 @@ class DeepseekV4HipRadixBackend( block_q=_GROUPED_ASM_BLOCK_Q, ) + u = core.unified + u.grp_qo_indptr = None + u.grp_hca_indices = u.grp_hca_indptr = u.grp_hca_bounds = None + u.grp_swa_indices = u.grp_swa_indptr = u.grp_swa_bounds = None + num_reqs = N // num_draft if num_draft > 1 and N % num_draft == 0 else 0 + if self.is_dspark and num_reqs and is_unified_kv_fp8(): + grouped = functools.partial( + runtime.build_grouped_verify_streams, + state_slot, + core.positions_casual, + num_draft=num_draft, + win=pool.unified_swa_window, + ring_stride=pool.unified_swa_ring_size, + swa_pages=pool.unified_swa_pages, + ) + pays = functools.partial( + runtime.flydsl_grouped_verify_pays, + self._fp8_decode_num_heads, + num_draft, + num_reqs, + epi=self._fp8_verify_epi, + ) + if pays(128): + ( + u.grp_hca_indices, + u.grp_hca_indptr, + u.grp_qo_indptr, + u.grp_hca_bounds, + ) = grouped(hca_len, hca_page_indices) + if pays(0): + ( + u.grp_swa_indices, + u.grp_swa_indptr, + u.grp_qo_indptr, + u.grp_swa_bounds, + ) = grouped(None, None) + # SWA ring write target, same value for every layer this forward. req_slot = state_slot.to(torch.int64) core.unified.swa_loc = ( @@ -2574,9 +2678,12 @@ class DeepseekV4HipRadixBackend( save_kv_cache: bool = True, q_rope: Optional[torch.Tensor] = None, k_rope: Optional[torch.Tensor] = None, - ) -> torch.Tensor: + wo_a_quant_freqs: Optional[torch.Tensor] = None, + ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: """q_rope present: packed fp8 q over the two-pool fp8 layout (asm decode; - prefill also needs k_rope). Absent: plain bf16 q and pool (Triton).""" + prefill also needs k_rope). Absent: plain bf16 q and pool (Triton). + wo_a_quant_freqs (fp8 decode): return the inverse-RoPE'd wo_a mxfp8 + operands instead of the attention output.""" from sglang.kernels.ops.attention.dsv4.unified_kv_kernels import runtime pool = self.token_to_kv_pool @@ -2650,9 +2757,44 @@ class DeepseekV4HipRadixBackend( "the v4 nm asm kernel hardcodes 1/sqrt(512), this backend is " f"at {self.softmax_scale}" ) + qo_indptr, max_seqlen_q, q_kv_bounds = None, 1, None + grp = { + 128: ( + unified_metadata.grp_hca_indices, + unified_metadata.grp_hca_indptr, + unified_metadata.grp_hca_bounds, + ), + 0: ( + unified_metadata.grp_swa_indices, + unified_metadata.grp_swa_indptr, + unified_metadata.grp_swa_bounds, + ), + }.get(compress_ratio) + if verify_as_decode and grp is not None and grp[0] is not None: + kv_indices, kv_indptr, q_kv_bounds = grp + qo_indptr = unified_metadata.grp_qo_indptr + max_seqlen_q = self.target_verify_num_draft_tokens + if wo_a_quant_freqs is not None: + return runtime.decode_fp8_2buff_wo_a_mxfp8( + q=q, + q_rope=q_rope, + unified_kv=unified, + unified_kv_rope=pool.get_unified_kv_rope(layer_id), + kv_indices=kv_indices, + kv_indptr=kv_indptr, + attn_sink=attn_sink, + positions=positions, + rope_freqs=wo_a_quant_freqs, + compress_ratio=compress_ratio, + qo_indptr=qo_indptr, + max_seqlen_q=max_seqlen_q, + q_kv_bounds=q_kv_bounds, + ) gasm = ( unified_metadata.gasm_indices - if compress_ratio == 128 and q.shape[1] == _GROUPED_ASM_GQA + if compress_ratio == 128 + and q.shape[1] == _GROUPED_ASM_GQA + and q_kv_bounds is None else None ) if gasm is not None: @@ -2679,6 +2821,14 @@ class DeepseekV4HipRadixBackend( attn_sink=attn_sink, v_head_dim=layer.v_head_dim, compress_ratio=compress_ratio, + qo_indptr=qo_indptr, + max_seqlen_q=max_seqlen_q, + q_kv_bounds=q_kv_bounds, + kv_len_hint=( + self._fp8_split_kv_hint.get(compress_ratio) + if qo_indptr is None + else None + ), ) from sglang.kernels.ops.attention.dsv4.unified_kv_kernels.paged_decode import ( _kv_splits_for_stream, @@ -3034,8 +3184,9 @@ class DeepseekV4HipRadixBackend( q_rope: Optional[torch.Tensor] = None, k_rope: Optional[torch.Tensor] = None, inv_rope: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + wo_a_quant_freqs: Optional[torch.Tensor] = None, **_, - ) -> torch.Tensor: + ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: """inv_rope=(freqs_real, positions) has the kernel apply the inverse RoPE to the last freqs_real.shape[-1] dims of every output head.""" if self.mtp_enabled and forward_batch.forward_mode.is_idle(): @@ -3066,6 +3217,7 @@ class DeepseekV4HipRadixBackend( save_kv_cache=save_kv_cache, q_rope=q_rope, k_rope=k_rope, + wo_a_quant_freqs=wo_a_quant_freqs, ) assert inv_rope is None, ( "the unified_kv model path applies its inverse RoPE" diff --git a/python/sglang/srt/models/deepseek_common/amd/deepseek_v4_wo_a_fp8.py b/python/sglang/srt/models/deepseek_common/amd/deepseek_v4_wo_a_fp8.py index a50f1365d2..ddbbb16309 100644 --- a/python/sglang/srt/models/deepseek_common/amd/deepseek_v4_wo_a_fp8.py +++ b/python/sglang/srt/models/deepseek_common/amd/deepseek_v4_wo_a_fp8.py @@ -230,6 +230,24 @@ def apply_wo_a_fp8_mxscale_fused_invrope( ) +def apply_wo_a_fp8_mxscale_prequant( + o_fp8: torch.Tensor, + o_scale: torch.Tensor, + weight: torch.Tensor, + weight_scale: torch.Tensor, +) -> torch.Tensor: + """fp8 ``wo_a`` on ``quant_wo_a_act_mxfp8``-layout operands flattened per + token (fp8 ``[T, G*D]``, e8m0 ``[T, G*D/128]``). Returns bf16 ``[T, G, R]``.""" + T, G = o_fp8.shape[0], weight.shape[0] + return _batched_gemm_a8w8_mxscale( + o_fp8.view(T, G, -1), + weight, + o_scale.view(T, G, -1), + weight_scale, + dtype=torch.bfloat16, + ) + + def _is_power_of_two(scale: torch.Tensor) -> bool: """True when every fp32 scale is an exact power of two (zero mantissa).""" bits = scale.detach().float().contiguous().view(torch.int32) @@ -287,6 +305,7 @@ def wo_a_weight_scale_to_e8m0( __all__ = [ "WO_A_MXFP8_GROUP_SIZE", "apply_wo_a_fp8_mxscale", + "apply_wo_a_fp8_mxscale_prequant", "is_wo_a_fp8_mxscale_supported", "quant_wo_a_act_mxfp8", "wo_a_weight_scale_to_e8m0", diff --git a/python/sglang/srt/models/deepseek_v4.py b/python/sglang/srt/models/deepseek_v4.py index 78a5ae9df5..11ae8398aa 100644 --- a/python/sglang/srt/models/deepseek_v4.py +++ b/python/sglang/srt/models/deepseek_v4.py @@ -418,6 +418,7 @@ _wo_a_aiter_batched_gemm_disabled = False # ``None`` here means the platform keeps the bf16 absorb GEMM. _wo_a_fp8_mxscale = None _wo_a_fp8_mxscale_fused_invrope = None +_wo_a_fp8_mxscale_prequant = None _wo_a_weight_scale_to_e8m0 = None _hip = None if _is_hip: @@ -425,6 +426,7 @@ if _is_hip: from sglang.srt.models.deepseek_common.amd.deepseek_v4_wo_a_fp8 import ( apply_wo_a_fp8_mxscale, apply_wo_a_fp8_mxscale_fused_invrope, + apply_wo_a_fp8_mxscale_prequant, is_wo_a_fp8_fused_invrope_supported, is_wo_a_fp8_mxscale_supported, wo_a_weight_scale_to_e8m0, @@ -440,6 +442,7 @@ if _is_hip: and is_wo_a_fp8_fused_invrope_supported() ): _wo_a_fp8_mxscale_fused_invrope = apply_wo_a_fp8_mxscale_fused_invrope + _wo_a_fp8_mxscale_prequant = apply_wo_a_fp8_mxscale_prequant @functools.lru_cache(maxsize=1) @@ -1077,6 +1080,27 @@ class MqaAttentionBase(nn.Module): # Other FlashMLA implementations retain their existing padded shape. return 64 if self.n_local_heads <= 64 else self.n_heads + def _wo_a_quant_freqs( + self, forward_batch: ForwardBatch, q_rope: Optional[torch.Tensor] + ) -> Optional[torch.Tensor]: + """Rope table when the fp8 unified_kv decode (aiter FlyDSL) should also + do the inverse RoPE and the wo_a mxfp8 quant; else None.""" + mode = forward_batch.forward_mode + if q_rope is None or not (mode.is_decode() or mode.is_target_verify()): + return None + if not hasattr(self, "_wo_a_freqs"): + from sglang.kernels.ops.attention.dsv4.unified_kv_kernels import runtime + + fused = ( + _wo_a_fp8_mxscale_prequant is not None + and self.wo_a_fp8 + and runtime.flydsl_decode_supported(q_rope.shape[1]) + ) + self._wo_a_freqs = ( + torch.view_as_real(self.freqs_cis).flatten(1) if fused else None + ) + return self._wo_a_freqs + def _local_attn_sink(self, kernel_num_heads: Optional[int] = None) -> torch.Tensor: if self.attn_tp_size == 1: return self.attn_sink @@ -2559,6 +2583,7 @@ class MQALayer(MqaAttentionBase): if _is_hip else None ) + wo_a_quant_freqs = None if unified: # only the HIP radix backend takes these; passing them always would # leave non-ROCm depending on the **_ in its forward() to drop them, and @@ -2570,6 +2595,9 @@ class MQALayer(MqaAttentionBase): rope_kwargs["k_rope"] = k_rope if inv_rope is not None: rope_kwargs["inv_rope"] = inv_rope + wo_a_quant_freqs = self._wo_a_quant_freqs(forward_batch, q_rope) + if wo_a_quant_freqs is not None: + rope_kwargs["wo_a_quant_freqs"] = wo_a_quant_freqs o = attn_backend.forward( q=q_out if q_out is not None else q, k=attn_k, @@ -2613,7 +2641,15 @@ class MQALayer(MqaAttentionBase): **({} if inv_rope is None else {"inv_rope": inv_rope}), ) o = o[:, tp_slice, :] - if ( + if wo_a_quant_freqs is not None: + # the decode already returned the wo_a mxfp8 (codes, scales) + G = self.n_local_groups + o = _wo_a_fp8_mxscale_prequant( + *o, + self.wo_a.weight.view(G, self.o_lora_rank, -1), + self.wo_a.weight_scale_inv.data, + ) + elif ( self.wo_a_fp8 and _wo_a_fp8_mxscale_fused_invrope is not None and not _is_npu diff --git a/test/registered/e2e/dsv4/test_dsv4_unified_fp8_decode.py b/test/registered/e2e/dsv4/test_dsv4_unified_fp8_decode.py index d47345d628..93162cbb5c 100644 --- a/test/registered/e2e/dsv4/test_dsv4_unified_fp8_decode.py +++ b/test/registered/e2e/dsv4/test_dsv4_unified_fp8_decode.py @@ -1,4 +1,5 @@ -"""Decode read path over the two-pool fp8 unified_kv (aiter's v4 nm asm kernel). +"""Decode read path over the two-pool fp8 unified_kv (aiter's v4 nm decode, +FlyDSL or asm). What these pin is the reader-side plumbing, not the kernel's arithmetic: the packed 512 B nope row and the bf16 rope pool addressed by one shared row index, @@ -13,18 +14,25 @@ The quantization helpers mirror aiter's own reference file is a test, not part of the aiter package. """ +import contextlib +import functools +import itertools import unittest +from unittest import mock import torch +from sglang.kernels.ops.attention.dsv4 import fused_rope_inplace from sglang.kernels.ops.attention.dsv4.unified_kv_kernels import runtime +from sglang.srt.environ import envs from sglang.srt.mem_cache.deepseek_v4_memory_pool import DSV4_FP8_NOPE_ROW_BYTES +from sglang.srt.models.deepseek_common.amd import deepseek_v4_wo_a_fp8 as wo_a_fp8 from sglang.srt.utils import is_gfx95_supported, is_hip from sglang.test.ci.ci_register import register_amd_ci from sglang.test.test_utils import CustomTestCase # the asm shader is only shipped for gfx950 -register_amd_ci(est_time=60, suite="stage-b-test-1-gpu-small-amd-mi35x") +register_amd_ci(est_time=180, suite="stage-b-test-1-gpu-small-amd-mi35x") DEVICE = torch.device("cuda") @@ -44,6 +52,26 @@ _needs_gfx950 = unittest.skipUnless( ) +_SPLIT_MODEL_ONLY = dict( + SGLANG_DSV4_FP8_DECODE_SPLIT_MODEL=True, SGLANG_DSV4_FP8_HCA_PERSIST=False +) +_PS_ONLY = dict( + SGLANG_DSV4_FP8_DECODE_SPLIT_MODEL=False, SGLANG_DSV4_FP8_HCA_PERSIST=True +) + + +@contextlib.contextmanager +def _asm_env(**env): + """asm decode (FlyDSL off) with the given switches""" + with contextlib.ExitStack() as stack: + stack.enter_context( + mock.patch.object(runtime, "_flydsl_v4_decode", lambda: None) + ) + for name, value in env.items(): + stack.enter_context(getattr(envs, name).override(value)) + yield + + def _pow2_ceil_scale(amax: torch.Tensor) -> torch.Tensor: """amax/fp8_max -> the next power of two at or above it, as fp32""" return torch.pow(2.0, torch.clamp_min(amax, 1e-4).log2().ceil()).to(torch.float32) @@ -142,23 +170,29 @@ class TestUnifiedFp8Decode(CustomTestCase): torch.manual_seed(7) self.rows = 256 - def _run(self, lengths, num_heads): + def _run(self, lengths, num_heads, asm=False): T = len(lengths) pool_nope, pool_rope, kv_silver = _make_latent(self.rows) q_packed, q_rope, q_silver = _make_latent(T, num_heads) indices, indptr = _ragged(lengths, self.rows) sink = torch.randn(num_heads, device=DEVICE, dtype=torch.float32) - got = runtime.decode_fp8_2buff( - q=q_packed, - q_rope=q_rope, - unified_kv=pool_nope, - unified_kv_rope=pool_rope, - kv_indices=indices, - kv_indptr=indptr, - attn_sink=sink, - v_head_dim=V_HEAD_DIM, - ) + with ( + mock.patch.object(runtime, "_flydsl_v4_decode", lambda: None) + if asm + # FlyDSL at H128 is opt-in; cover it here + else envs.SGLANG_DSV4_FLYDSL_DECODE_H128.override(True) + ): + got = runtime.decode_fp8_2buff( + q=q_packed, + q_rope=q_rope, + unified_kv=pool_nope, + unified_kv_rope=pool_rope, + kv_indices=indices, + kv_indptr=indptr, + attn_sink=sink, + v_head_dim=V_HEAD_DIM, + ) want = _reference(q_silver, kv_silver, indices, indptr, sink) return got.float(), want @@ -186,9 +220,105 @@ class TestUnifiedFp8Decode(CustomTestCase): self._assert_close(got, want) @_needs_gfx950 - def test_head_count_64(self): - got, want = self._run([48, 96], num_heads=64) - self._assert_close(got, want) + def test_head_counts_both_backends(self): + for num_heads in (16, 32, 64, 128): + for asm in (False, True): + with self.subTest(num_heads=num_heads, asm=asm): + got, want = self._run([48, 96, 1, 250], num_heads, asm=asm) + self._assert_close(got, want) + + @_needs_gfx950 + @unittest.skipUnless( + runtime.flydsl_decode_supported(16, 7, grouped=True), + "aiter has no FlyDSL v4 decode", + ) + def test_grouped_verify_streams_match_per_token(self): + """Each draft of a shared per-request stream matches its per-token + stream, incl. drafts straddling a c128 commit and short requests.""" + win, ring = 128, 256 + num_draft, reqs, rows = 7, 9, 8192 + pool_nope, pool_rope, _ = _make_latent(rows) + swa_pages = reqs * ring + last = torch.tensor([6, 70, 127, 128, 133, 900, 3000, 4000, 262], device=DEVICE) + positions = ( + last.repeat_interleave(num_draft) + - num_draft + + 1 + + torch.arange(num_draft, device=DEVICE).repeat(reqs) + ).clamp_min(0) + state_slot = torch.arange(reqs, device=DEVICE).repeat_interleave(num_draft) + N = positions.numel() + width = 40 + hca_len = torch.div(positions + 1, 128, rounding_mode="floor").to(torch.int32) + tail_rows = torch.randperm(rows - swa_pages, device=DEVICE)[: reqs * width] + tails = tail_rows.view(reqs, width).to(torch.int32) + hca_pages = tails.repeat_interleave(num_draft, dim=0).contiguous() + swa_len = torch.clamp(positions + 1, max=win).to(torch.int32) + zero = torch.zeros(N, dtype=torch.int32, device=DEVICE) + swa_i, swa_p, hca_i, hca_p, _, _ = runtime.build_decode_streams( + state_slot=state_slot.to(torch.int32), + positions=positions.to(torch.int32), + swa_len=swa_len, + hca_len=hca_len, + csa_len=zero, + hca_page_indices=hca_pages, + csa_width=0, + win=win, + ring_stride=ring, + swa_pages=swa_pages, + ) + q_packed, q_rope, _ = _make_latent(N, 16) + sink = torch.randn(16, device=DEVICE, dtype=torch.float32) + angle = torch.rand(4096, ROPE_DIM // 2, device=DEVICE) * 6.28 + freqs = torch.view_as_real(torch.polar(torch.ones_like(angle), angle)).flatten( + 1 + ) + kw = dict( + q=q_packed, + q_rope=q_rope, + unified_kv=pool_nope, + unified_kv_rope=pool_rope, + attn_sink=sink, + ) + for cr, tail, tok in ( + (128, (hca_len, hca_pages), (hca_i, hca_p)), + (0, (None, None), (swa_i, swa_p)), + ): + g_idx, g_kvp, g_qop, bnd = runtime.build_grouped_verify_streams( + state_slot, + positions, + *tail, + num_draft=num_draft, + win=win, + ring_stride=ring, + swa_pages=swa_pages, + ) + grp = dict( + kv_indices=g_idx, + kv_indptr=g_kvp, + qo_indptr=g_qop, + max_seqlen_q=num_draft, + q_kv_bounds=bnd, + compress_ratio=cr, + ) + with self.subTest(compress_ratio=cr, mode="bf16"): + want = runtime.decode_fp8_2buff( + **kw, kv_indices=tok[0], kv_indptr=tok[1], v_head_dim=V_HEAD_DIM + ) + got = runtime.decode_fp8_2buff(**kw, **grp, v_head_dim=V_HEAD_DIM) + self._assert_close(got.float(), want.float(), atol=1e-2, rtol=1e-2) + with self.subTest(compress_ratio=cr, mode="fused"): + fused = dict(positions=positions.to(torch.int64), rope_freqs=freqs) + want = runtime.decode_fp8_2buff_wo_a_mxfp8( + **kw, + **fused, + kv_indices=tok[0], + kv_indptr=tok[1], + compress_ratio=cr, + ) + got = runtime.decode_fp8_2buff_wo_a_mxfp8(**kw, **fused, **grp) + w, g = _dequant_wo_a(*want), _dequant_wo_a(*got) + self.assertLess(((g - w).norm() / w.norm()).item(), 2e-2) @_needs_gfx950 def test_cuda_graph_pad_reads_only_the_reserved_ring_row(self): @@ -262,9 +392,138 @@ class TestUnifiedFp8Decode(CustomTestCase): """ lengths = [200, 64] * 24 # 48 tokens, both layer flavours' segment lengths self.assertGreater(len(lengths), 40) - got, want = self._run(lengths, num_heads=16) + got, want = self._run(lengths, num_heads=16, asm=True) self._assert_close(got, want) + def test_ported_split_model_matches_aiter(self): + """the port is aiter's gfx950 model verbatim; where aiter ships the + helper the two agree on every shape, else pin known picks""" + port = functools.partial(runtime._ported_num_kv_splits, gfx="gfx950") + known = {(24, 1152): 4, (48, 1152): 2, (96, 1152): 1, (192, 1152): 1} + for (n, kv), s in known.items(): + self.assertEqual(port(n, 128, kv, cu_num=256), s, (n, kv)) + self.assertIsNone(runtime._ported_num_kv_splits(96, 128, 1152, "gfx942", 304)) + try: + from aiter.mla import get_mla_v4_nm_num_kv_splits + except ImportError: + return + if not (torch.cuda.is_available() and is_gfx95_supported()): + return + cu = runtime._cu_count() + for n, h, kv in itertools.product( + (1, 7, 24, 48, 96, 128, 192, 384), (16, 128), (128, 1152, 1690, 8320) + ): + self.assertEqual( + port(n, h, kv, cu_num=cu), get_mla_v4_nm_num_kv_splits(n, h, kv) + ) + + def _run_plan(self, lengths, num_heads, kv_len_hint, compress_ratio, **env): + T = len(lengths) + rows = max(self.rows, max(lengths) + 1) + pool_nope, pool_rope, kv_silver = _make_latent(rows) + q_packed, q_rope, q_silver = _make_latent(T, num_heads) + indices, indptr = _ragged(lengths, rows) + sink = torch.randn(num_heads, device=DEVICE, dtype=torch.float32) + kw = dict( + q=q_packed, + q_rope=q_rope, + unified_kv=pool_nope, + unified_kv_rope=pool_rope, + kv_indices=indices, + kv_indptr=indptr, + attn_sink=sink, + v_head_dim=V_HEAD_DIM, + compress_ratio=compress_ratio, + kv_len_hint=kv_len_hint, + ) + with _asm_env(**env): + got = runtime.decode_fp8_2buff(**kw) + want = _reference(q_silver, kv_silver, indices, indptr, sink) + return got.float(), want, kw + + @_needs_gfx950 + def test_cost_model_split_plan_matches_reference(self): + """H128 DP-attention rows, CSA/SWA-shaped KV, the split count from the + latency model and a slice of the constant split table""" + gen = torch.Generator().manual_seed(3) + for T, (cr, hint) in itertools.product( + (24, 48, 96, 128, 192), ((4, 1152), (0, 128)) + ): + lengths = torch.randint(1, hint + 1, (T,), generator=gen).tolist() + with self.subTest(T=T, compress_ratio=cr): + plan = runtime.fp8_decode_split_plan(T, 128, hint, DEVICE) + self.assertIsNotNone(plan) + s, split_indptr = plan + self.assertTrue( + torch.equal( + split_indptr.cpu(), + torch.arange(T + 1, dtype=torch.int32) * s, + ) + ) + got, want, _ = self._run_plan( + lengths, 128, hint, cr, **_SPLIT_MODEL_ONLY + ) + self._assert_close(got, want) + + @_needs_gfx950 + def test_constant_split_table_graph_replay(self): + """capture with the plan, rewrite the KV ranges, replay: matches eager""" + T, hint = 96, 1152 + lengths = [hint - 7 * i for i in range(T)] + _, _, kw = self._run_plan(lengths, 128, hint, 4, **_SPLIT_MODEL_ONLY) + self._graph_replay_matches_eager(kw, **_SPLIT_MODEL_ONLY) + + def _graph_replay_matches_eager(self, kw, **env): + with _asm_env(**env): + stream = torch.cuda.Stream() + stream.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(stream): + runtime.decode_fp8_2buff(**kw) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph, stream=stream): + captured = runtime.decode_fp8_2buff(**kw) + # new, shorter ragged ranges in the same buffers + indptr = kw["kv_indptr"] + T = indptr.numel() - 1 + shortest = int((indptr[1:] - indptr[:-1]).min()) + new_len = torch.randint(1, shortest + 1, (T,), device=DEVICE) + indptr[1:] = torch.cumsum(new_len, 0).to(torch.int32) + graph.replay() + torch.cuda.synchronize() + eager = runtime.decode_fp8_2buff(**kw) + self._assert_close(captured.float(), eager.float(), atol=1e-2, rtol=1e-2) + + @_needs_gfx950 + @unittest.skipUnless(runtime._hca_ps_api() is not None, "aiter has no v4 nm ps") + def test_hca_persistent_matches_reference(self): + """H128 HCA rows on the persistent kernel: uneven KV (AgentX-like + dispersion), including cuda-graph pad rows of length 1""" + gen = torch.Generator().manual_seed(5) + for T in (16, 96, 192): + lengths = torch.randint(129, 1700, (T,), generator=gen).tolist() + lengths[-3:] = [1, 1, 1] + with self.subTest(T=T): + got, want, _ = self._run_plan(lengths, 128, 1690, 128, **_PS_ONLY) + self.assertIn(runtime._dev_key(DEVICE), runtime._hca_ps_workspaces) + self._assert_close(got, want) + + @_needs_gfx950 + @unittest.skipUnless(runtime._hca_ps_api() is not None, "aiter has no v4 nm ps") + def test_hca_persistent_graph_replay_varying_kv(self): + lengths = [1690 - 11 * i for i in range(128)] + _, _, kw = self._run_plan(lengths, 128, 1690, 128, **_PS_ONLY) + self._graph_replay_matches_eager(kw, **_PS_ONLY) + + @_needs_gfx950 + @unittest.skipUnless(runtime._hca_ps_api() is not None, "aiter has no v4 nm ps") + def test_hca_persistent_matches_split_kernel(self): + """same inputs, ps vs the asm split kernel""" + lengths = [1500, 300, 129, 1690, 1] * 20 + got_ps, _, kw = self._run_plan(lengths, 128, 1690, 128, **_PS_ONLY) + with _asm_env(SGLANG_DSV4_FP8_HCA_PERSIST=False): + got_asm = runtime.decode_fp8_2buff(**kw).float() + self._assert_close(got_ps, got_asm, atol=1e-2, rtol=1e-2) + @_needs_gfx950 def test_rejects_pool_that_is_not_a_pair(self): pool_nope, pool_rope, _ = _make_latent(self.rows) @@ -305,6 +564,80 @@ class TestUnifiedFp8Decode(CustomTestCase): v_head_dim=V_HEAD_DIM, ) + @_needs_gfx950 + @unittest.skipUnless( + runtime.flydsl_decode_supported(16), + "no FlyDSL v4 nm decode (aiter or in-tree)", + ) + def test_fused_wo_a_operands_match_unfused_chain(self): + """decode + inverse RoPE + wo_a quant in one launch == the unfused chain, + eagerly and from a replayed cuda graph (TP8 H16 and DP attention H128).""" + max_pos = 4096 + angle = torch.rand(max_pos, ROPE_DIM // 2, device=DEVICE) * 6.28 + freqs_cis = torch.polar(torch.ones_like(angle), angle) + pool_nope, pool_rope, _ = _make_latent(self.rows) + for H, (lengths, compress_ratio) in itertools.product( + (16, 128), (([64, 17, 128, 1] * 3, 4), ([200] * 40, 128)) + ): + T = len(lengths) + q_packed, q_rope, _ = _make_latent(T, H) + indices, indptr = _ragged(lengths, self.rows) + kw = dict( + q=q_packed, + q_rope=q_rope, + unified_kv=pool_nope, + unified_kv_rope=pool_rope, + kv_indices=indices, + kv_indptr=indptr, + attn_sink=torch.randn(H, device=DEVICE, dtype=torch.float32), + ) + positions = torch.randint(0, max_pos, (T,), device=DEVICE) + + out = runtime.decode_fp8_2buff(**kw, v_head_dim=V_HEAD_DIM) + fused_rope_inplace( + out[..., -ROPE_DIM:], None, freqs_cis, positions=positions, inverse=True + ) + want = _dequant_wo_a( + *wo_a_fp8.quant_wo_a_act_mxfp8(out.view(T, H // 8, -1)) + ) + + def fused(): + return runtime.decode_fp8_2buff_wo_a_mxfp8( + **kw, + positions=positions, + rope_freqs=torch.view_as_real(freqs_cis).flatten(1), + compress_ratio=compress_ratio, + ) + + stream = torch.cuda.Stream() + stream.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(stream): # warm up on the capture stream + eager = fused() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph, stream=stream): + captured = fused() + graph.replay() + torch.cuda.synchronize() + + with self.subTest(H=H, T=T, compress_ratio=compress_ratio): + got = _dequant_wo_a(*eager) + err = (got - want).norm() / want.norm() + self.assertLess(err.item(), 2e-2) + self.assertTrue( + torch.equal( + captured[0].view(torch.uint8), eager[0].view(torch.uint8) + ) + ) + self.assertTrue(torch.equal(captured[1], eager[1])) + + +def _dequant_wo_a(codes: torch.Tensor, scales: torch.Tensor) -> torch.Tensor: + """wo_a mxfp8 operands (any [T, ...] row-major view) -> fp32 [T, H*512]""" + T = codes.shape[0] + blocks = codes.reshape(T, -1, 128).float() + scale = torch.exp2(scales.reshape(T, -1, 1).float() - 127) + return (blocks * scale).reshape(T, -1) + if __name__ == "__main__": unittest.main() diff --git a/test/registered/kernels/ops/attention/dsv4/test_flydsl_v4nm_decode.py b/test/registered/kernels/ops/attention/dsv4/test_flydsl_v4nm_decode.py new file mode 100644 index 0000000000..20fc92b62a --- /dev/null +++ b/test/registered/kernels/ops/attention/dsv4/test_flydsl_v4nm_decode.py @@ -0,0 +1,552 @@ +"""In-tree FlyDSL v4 nm MLA decode (ROCm/aiter#5882 copy, gfx950) vs an fp64 reference. + +Ported from ROCm/aiter#5882 ``op_tests/test_flydsl_mla_v4_decode.py`` (commit +03359c43df), trimmed to the DeepSeek-V4 shapes sglang runs: 16 / 32 / 64 / 128 +local heads at one q row (TP8 / TP4 / TP2 / DP attention), 16 heads at four q +rows (grouped asm verify blocks), the fused inverse-RoPE + wo_a mxfp8 epilogue, +and grouped DSpark verify (q_kv_bounds) at 4 and 7 drafts. The asm comparison +and perf tables of the original are dropped; the assertions are kept. +Rows are packed like the live KV pool, 0xFF padding at bytes 462..511. + +TODO(aiter#5882): remove once ROCm/aiter#5882 is in the aiter pin +""" + +from __future__ import annotations + +import itertools +import sys + +import pytest +import torch + +from sglang.srt.utils import is_gfx95_supported, is_hip +from sglang.test.ci.ci_register import register_amd_ci + +register_amd_ci(est_time=600, stage="stage-b", runner_config="1-gpu-small-amd-mi35x") + +if not (is_hip() and is_gfx95_supported()): + pytest.skip("FlyDSL v4 nm decode is gfx950-only", allow_module_level=True) + +try: + from sglang.kernels.ops.attention.dsv4 import flydsl_v4nm + from sglang.kernels.ops.attention.dsv4.flydsl_v4nm import mla_v4_decode as fly_v4 +except ImportError as e: # no flydsl, or an aiter before ROCm/aiter#6377 + pytest.skip( + f"in-tree FlyDSL v4 nm decode unavailable: {e}", allow_module_level=True + ) + +flydsl_mla_decode_fwd_v4_nm = flydsl_v4nm.flydsl_mla_decode_fwd_v4_nm +flydsl_mla_v4_decode_fused = flydsl_v4nm.flydsl_mla_v4_decode_fused +flydsl_mla_v4_decode_supported = flydsl_v4nm.flydsl_mla_v4_decode_supported +flydsl_mla_v4_grouped_verify_meta = flydsl_v4nm.flydsl_mla_v4_grouped_verify_meta + +FP8, BF16, I32 = torch.float8_e4m3fn, torch.bfloat16, torch.int32 +NOPE, ROPE, DV, ROW, QB = 448, 64, 512, 512, 128 +PAD = NOPE + 14 +WIN, RING, CRATIO = 128, 256, 128 # SWA window, ring slots per request, HCA ratio + + +@pytest.fixture(autouse=True) +def _cuda_default_device(): + with torch.device("cuda"): + yield + + +def pack_rows(n, gen, pad=0xFF): + """fp8 rows [n, 512] u8 (448 e4m3 + 7 e8m0 scales, each twice) + bf16 rope.""" + nope = torch.randn(n, NOPE // 64, 64, generator=gen) + rope = torch.randn(n, ROPE, generator=gen).to(BF16) + scale = torch.exp2(torch.ceil(torch.log2((nope.abs().amax(-1) / 448).clamp(1e-4)))) + row = torch.full((n, ROW), pad, dtype=torch.uint8) + row[:, :NOPE] = (nope / scale[..., None]).to(FP8).view(n, NOPE).view(torch.uint8) + e8 = (torch.log2(scale).to(torch.int32) + 127).to(torch.uint8) + row[:, NOPE:PAD] = e8.repeat_interleave(2, dim=-1) + return row, rope + + +def dequant_rows(row, rope): + vals = row[..., :NOPE].view(FP8).double() + sc = torch.exp2(row[..., NOPE:PAD:2].double() - 127) + return torch.cat( + [(vals.unflatten(-1, (7, 64)) * sc[..., None]).flatten(-2), rope.double()], -1 + ) + + +def make_inputs(H, msq, batch, kv_len, ragged, seed=0): + """batch sequences of msq q rows; ragged: kv lengths in [0, 2 kv_len], every 5th + empty (2: the first one 16 kv_len).""" + gen = torch.Generator(device="cuda").manual_seed(seed) + gcpu = torch.Generator().manual_seed(seed) + kv_lens = [kv_len] * batch + if ragged: + kv_lens = torch.randint( + 0, 2 * kv_len + 1, (batch,), generator=gcpu, device="cpu" + ) + kv_lens = [0 if b % 5 == 3 else n for b, n in enumerate(kv_lens.tolist())] + kv_lens[0] = 16 * kv_len if ragged == 2 else kv_lens[0] + rows = max(1024, sum(kv_lens) + 64) + kv, kv_rope = pack_rows(rows, gen) + kv_indptr = torch.tensor([0] + kv_lens).cumsum(0).to(I32) + kv_indices = torch.randperm(rows, generator=gen)[: max(1, sum(kv_lens))].to(I32) + q, q_rope = pack_rows(batch * msq * H, gen) + return { + "q": q.view(-1, H, ROW), + "q_rope": q_rope.view(-1, H, ROPE), + "kv": kv, + "kv_rope": kv_rope, + "qo_indptr": torch.arange(batch + 1, dtype=I32) * msq, + "kv_indptr": kv_indptr, + "kv_indices": kv_indices, + "sink": torch.randn(H, generator=gen), + }, kv_lens + + +def run_torch(q, q_rope, kv, kv_rope, qo_indptr, kv_indptr, kv_indices, sink, **_): + """fp64 softmax([q k^T / sqrt(512), sink]) @ kv per sequence; rows without keys NaN.""" + qd = dequant_rows(q, q_rope) + out = torch.full(qd.shape, float("nan"), dtype=torch.float64) + qo, ki = qo_indptr.tolist(), kv_indptr.tolist() + for b in range(len(qo) - 1): + rows = kv_indices[ki[b] : ki[b + 1]].long() + if rows.numel() == 0 or qo[b + 1] == qo[b]: + continue + kd = dequant_rows(kv[rows], kv_rope[rows]) + s = torch.einsum("qhd,kd->qhk", qd[qo[b] : qo[b + 1]], kd) * DV**-0.5 + sk = sink.double()[None, :, None].expand(s.shape[0], -1, 1) + p = torch.softmax(torch.cat([s, sk], -1), -1)[..., :-1] + out[qo[b] : qo[b + 1]] = torch.einsum("qhk,kd->qhd", p, kd) + return out + + +def inv_rope(ref, positions, freqs): + T, H = ref.shape[:2] + fc = torch.view_as_complex(freqs[positions].double().view(T, 1, ROPE // 2, 2)) + tail = torch.view_as_complex(ref[..., NOPE:].contiguous().view(T, H, ROPE // 2, 2)) + out = ref.clone() + out[..., NOPE:] = torch.view_as_real(tail * fc.conj()).view(T, H, ROPE) + return out + + +def rope_freqs(max_pos=1 << 18): + inv = 10000.0 ** (-torch.arange(0, ROPE, 2, dtype=torch.float64) / ROPE) + ang = torch.arange(max_pos, dtype=torch.float64)[:, None] * inv + return torch.stack([ang.cos(), ang.sin()], -1).flatten(1).float() + + +def dequant_mxfp8(xq, xs): + T = xq.shape[0] + x = xq.view(FP8).view(T, -1, QB).double() + return x * torch.exp2(xs.view(torch.uint8).view(T, -1, 1).double() - 127) + + +def mxfp8_rel_l2(ref, xq, xs): + blk = ref.reshape(ref.shape[0], -1, QB) + return ((dequant_mxfp8(xq, xs) - blk).norm() / blk.norm()).item() + + +def rel_l2(o, ref): + return ((o.double() - ref).norm() / ref.norm()).item() + + +def decode(inp, out, msq, **kw): + rows = inp["kv"].shape[0] + return flydsl_mla_decode_fwd_v4_nm( + inp["q"].view(FP8), + inp["q_rope"], + inp["kv"].view(FP8).view(rows, 1, 1, ROW), + inp["kv_rope"].view(rows, 1, 1, ROPE), + out, + inp["qo_indptr"], + inp["kv_indptr"], + inp["kv_indices"], + msq, + sink=inp["sink"], + **kw, + ) + + +def graph_replay(fn, n=3, st=None): + """Warm up eagerly on a side stream (as sglang does), capture, replay n times.""" + st = st or torch.cuda.Stream() + st.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(st): + fn() + torch.cuda.current_stream().wait_stream(st) + g = torch.cuda.CUDAGraph() + with torch.cuda.graph(g, stream=st): + fn() + for _ in range(n): + g.replay() + torch.cuda.synchronize() + return g + + +def bits(x): + return x.view(torch.int16) if x.element_size() == 2 else x.view(torch.uint8) + + +# DeepSeek-V4 shapes: (H, msq, hints) x batches x kv lengths x ragged; sglang always +# passes a stream hint (csa / hca / swa), which only H16 msq1 plans read. +_BATCHES = (1, 9, 37, 129, 300) +_DECODE_CASES = [ + (H, msq, batch, kv_len, ragged, hint) + for (H, msq, hints) in ( + (16, 1, ("csa", "hca", "swa")), + (16, 4, ("hca",)), + (32, 1, ("hca",)), + (64, 1, ("hca",)), + (128, 1, ("hca",)), + ) + for batch, kv_len, ragged, hint in itertools.product( + _BATCHES, (128, 1152), (0, 1), hints + ) +] + [ + (16, 1, 37, 4096, 0, "csa"), + (128, 1, 129, 4096, 0, "hca"), + # one long stream among short ones: the key-balanced grid at short runs + (16, 1, 96, 256, 2, "hca"), +] + + +@pytest.mark.parametrize("H, msq, batch, kv_len, ragged, hint", _DECODE_CASES) +def test_decode(H, msq, batch, kv_len, ragged, hint): + inp, kv_lens = make_inputs(H, msq, batch, kv_len, ragged) + ref = run_torch(**inp) + ok = torch.tensor( + [b * msq + r for b, n in enumerate(kv_lens) if n for r in range(msq)], + dtype=torch.long, + ) + empty = torch.tensor( + [b * msq + r for b, n in enumerate(kv_lens) if not n for r in range(msq)], + dtype=torch.long, + ) + out = torch.empty((batch * msq, H, DV), dtype=BF16) + decode(inp, out, msq, hint=hint) + assert rel_l2(out[ok], ref[ok]) < 3e-3 + assert torch.isnan(out[empty].float()).all(), "kv_len == 0 rows must be NaN" + base = out.clone() + p = dict(inp, q=inp["q"].clone(), kv=inp["kv"].clone()) + for k in ("q", "kv"): # padding bytes are never read + p[k][..., PAD:] = torch.randint( + 0, 256, p[k][..., PAD:].shape, dtype=torch.uint8 + ) + decode(p, out, msq, hint=hint) + assert torch.equal(bits(out[ok]), bits(base[ok])) + out.zero_() + graph_replay(lambda: decode(inp, out, msq, hint=hint)) + assert torch.equal(bits(out), bits(base)), "graph replay" + + +_FUSED_CASES = list( + itertools.product( + ( + (16, 4, 1152), + (16, 128, 659), + (16, 0, 128), + (32, 128, 659), + (128, 128, 659), + (128, 4, 1152), + ), + (3, 40, 100, 512), + (0, 1), + ) +) + + +@pytest.mark.parametrize("shape, T, ragged", _FUSED_CASES) +def test_decode_fused(shape, T, ragged): + H, compress_ratio, kv_len = shape + inp, kv_lens = make_inputs(H, 1, T, kv_len, ragged) + del inp["qo_indptr"] + inp["positions"] = ( + 65536 + + torch.randint( + 0, 4096, (T,), generator=torch.Generator().manual_seed(2), device="cpu" + ).cuda() + ) + inp["freqs"] = rope_freqs() + ref = inv_rope( + run_torch(**inp, qo_indptr=torch.arange(T + 1, dtype=I32)), + inp["positions"], + inp["freqs"], + ) + ok = torch.tensor([t for t, n in enumerate(kv_lens) if n], dtype=torch.long) + empty = torch.tensor([t for t, n in enumerate(kv_lens) if not n], dtype=torch.long) + res = {} + + def fused(): + res["x"] = flydsl_mla_v4_decode_fused(**inp, compress_ratio=compress_ratio) + return res["x"] + + q8, s8 = [x.view(torch.uint8).clone() for x in fused()] + assert mxfp8_rel_l2(ref[ok], q8[ok], s8[ok]) < 0.05 + assert not (q8[ok] & 0x7F).eq(0x7F).any(), "NaN codes in xq" + assert ((q8[empty] & 0x7F) == 0x7F).all() and (s8[empty] == 0xFF).all() + graph_replay(fused) + assert torch.equal(res["x"][0].view(torch.uint8), q8) + assert torch.equal(res["x"][1], s8) + + +def make_dspark(num_draft, kind, requests, seed=0, H=16): + """One DSpark verify step on a pool [SWA ring: R x RING rows][compressed rows]: the + drafts of request r end at positions around the SWA window / block commits; per + token (sglang's streams): its window + (hca) its committed rows (p + 1) // 128.""" + gen = torch.Generator(device="cuda").manual_seed(seed) + base = [6, 10, 126, 127, 128, 131, 133, 135, 255, 256, 260, 4000, 70000, 131000] + last = [base[r % len(base)] + 7 * (r // len(base)) for r in range(requests)] + pos = [p - num_draft + 1 + d for p in last for d in range(num_draft)] + ncomp = [(p + 1) // CRATIO if kind == "hca" else 0 for p in last] + swa_pages = requests * RING + kv, kv_rope = pack_rows(swa_pages + max(1, sum(ncomp)), gen) + comp = torch.randperm(max(1, sum(ncomp)), generator=gen, device="cuda").tolist() + lists = [comp[sum(ncomp[:r]) : sum(ncomp[: r + 1])] for r in range(requests)] + tail_pages = torch.full((len(pos), max(1, max(ncomp))), -1, dtype=I32) + tail_len, toks = [], [] + for t, p in enumerate(pos): + r = t // num_draft + c = (p + 1) // CRATIO if kind == "hca" else 0 + tail_len.append(c) + tail_pages[t, :c] = torch.tensor(lists[r][:c], dtype=I32) + swa = [r * RING + x % RING for x in range(max(0, p - WIN + 1), p + 1)] + toks.append(swa + [x + swa_pages for x in lists[r][:c]]) + q, q_rope = pack_rows(len(pos) * H, gen) + hca = kind == "hca" + return { + "q": q.view(-1, H, ROW), + "q_rope": q_rope.view(-1, H, ROPE), + "kv": kv, + "kv_rope": kv_rope, + "sink": torch.randn(H, generator=gen, device="cuda"), + "meta": ( + torch.tensor([t // num_draft for t in range(len(pos))], dtype=I32), + torch.tensor(pos), + torch.tensor(tail_len, dtype=I32) if hca else None, + tail_pages if hca else None, + ), + "swa_pages": swa_pages, + "qo_indptr": torch.arange(len(pos) + 1, dtype=I32), + "kv_indptr": torch.tensor([0] + [len(x) for x in toks]).cumsum(0).to(I32), + "kv_indices": torch.tensor(list(itertools.chain(*toks)), dtype=I32), + } + + +def grouped_meta_ref(slot, pos, tail_len, tail_pages, win, ring_stride, swa_pages, nd): + """Torch reference of flydsl_mla_v4_grouped_verify_meta.""" + slot, pos = slot.tolist(), pos.tolist() + tl_ = tail_len.tolist() if tail_len is not None else [0] * len(pos) + idx, kvp, bnd = [], [0], [] + for g in range(len(pos) // nd): + last = g * nd + nd - 1 + p, c = pos[last], tl_[last] + n = min(p + 1, win + nd - 1) + if c: + idx += [ + x + swa_pages if x >= 0 else -1 for x in tail_pages[last, :c].tolist() + ] + idx += [ + slot[last] * ring_stride + (p - n + 1 + i) % ring_stride for i in range(n) + ] + s0 = p - n + 1 + for t in range(g * nd, last + 1): + bnd.append( + [0, tl_[t], c + max(0, pos[t] - win + 1 - s0), c + pos[t] - s0 + 1] + ) + kvp.append(len(idx)) + return ( + torch.tensor(idx, dtype=I32), + torch.tensor(kvp, dtype=I32), + torch.tensor(bnd, dtype=I32), + ) + + +@pytest.mark.parametrize( + "kind, num_draft, requests, epi", + list(itertools.product(("hca", "swa"), (4, 7), (14, 32, 72), ("bf16", "mxfp8"))), +) +def test_decode_grouped(kind, num_draft, requests, epi): + d = make_dspark(num_draft, kind, requests, seed=num_draft + requests) + N, H = d["q"].shape[:2] + ref = run_torch(**d) + idx, kvp, qop, bnd = flydsl_mla_v4_grouped_verify_meta( + *d["meta"], + win=WIN, + ring_stride=RING, + swa_pages=d["swa_pages"], + num_draft=num_draft, + ) + r_idx, r_kvp, r_bnd = grouped_meta_ref( + *d["meta"], WIN, RING, d["swa_pages"], num_draft + ) + L = int(r_kvp[-1]) + assert torch.equal(kvp, r_kvp) + assert torch.equal(bnd, r_bnd) + assert torch.equal(idx[:L], r_idx) + assert torch.equal(qop, torch.arange(requests + 1, dtype=I32) * num_draft) + grp = dict(d, qo_indptr=qop, kv_indptr=kvp, kv_indices=idx) + # a reversed interval (x1 < x0) is empty + b_rev, b_emp = bnd.clone(), bnd.clone() + b_rev[3] = torch.tensor([L + 7, 1, 97, 2], dtype=I32) + b_emp[3] = 0 + hint = "hca" if kind == "hca" else "swa" + if epi == "bf16": + out_g, out_t = [torch.empty((N, H, DV), dtype=BF16) for _ in range(2)] + + def grouped(b=bnd): + return decode(grp, out_g, num_draft, hint=hint, q_kv_bounds=b) + + grouped() + decode(d, out_t, 1, hint=hint) + assert rel_l2(out_g, ref) < 3e-3 + assert rel_l2(out_t, ref) < 3e-3 + g0 = out_g.clone() + e0 = grouped(b_emp).clone() + assert torch.isnan(e0[3].float()).all() + assert torch.equal(bits(grouped(b_rev)), bits(e0)) + out_g.zero_() + graph_replay(grouped) + assert torch.equal(bits(out_g), bits(g0)) + return + freqs = rope_freqs() + positions = d["meta"][1] + ref_f = inv_rope(ref, positions, freqs) + fa = (d["q"], d["q_rope"], d["kv"], d["kv_rope"]) + cr = 128 if kind == "hca" else 0 + res = {} + + def fused_grouped(b=bnd): + res["x"] = flydsl_mla_v4_decode_fused( + *fa, + kvp, + idx, + d["sink"], + positions, + freqs, + cr, + qo_indptr=qop, + max_seqlen_q=num_draft, + q_kv_bounds=b, + ) + return res["x"] + + xt = flydsl_mla_v4_decode_fused( + *fa, d["kv_indptr"], d["kv_indices"], d["sink"], positions, freqs, cr + ) + assert mxfp8_rel_l2(ref_f, *xt) < 0.05 + q8, s8 = [x.view(torch.uint8).clone() for x in fused_grouped()] + assert mxfp8_rel_l2(ref_f, q8, s8) < 0.05 + e0 = [x.view(torch.uint8).clone() for x in fused_grouped(b_emp)] + assert (e0[1][3] == 0xFF).all() + assert all( + torch.equal(x.view(torch.uint8), y) for x, y in zip(fused_grouped(b_rev), e0) + ) + graph_replay(fused_grouped) + assert torch.equal(res["x"][0].view(torch.uint8), q8) + assert torch.equal(res["x"][1], s8) + + +def test_contracts(): + """Unsupported pairs rejected; B = 0 is a no-op; > 4096 merge groups run unsplit; + graphs sharing one stream's workspace replay interleaved; a graph captured on a + fresh stream takes pre-zeroed protocol slots (no memset in the graph).""" + for H, msq in ((32, 2), (64, 4), (16, 3), (8, 1)): + assert not flydsl_mla_v4_decode_supported(H, msq) + for H in (16, 32, 64, 128): + assert flydsl_mla_v4_decode_supported(H, 1) + assert flydsl_mla_v4_decode_supported(16, 4) + assert flydsl_mla_v4_decode_supported(16, 7, grouped=True) + inp, _ = make_inputs(16, 1, 1, 128, False) + empty = dict( + inp, + q=inp["q"][:0], + q_rope=inp["q_rope"][:0], + qo_indptr=inp["qo_indptr"][:1], + kv_indptr=inp["kv_indptr"][:1], + ) + decode(empty, torch.empty((0, 16, DV), dtype=BF16), 1) + B = 4097 + assert fly_v4._plan_for(16, 1, B, "bf16", None, 256, False).S > 1 + assert fly_v4._plan(16, 1, B).S == 1 + inp, kv_lens = make_inputs(16, 2, B, 300, True) + inp = dict( + inp, q=inp["q"][::2].contiguous(), q_rope=inp["q_rope"][::2].contiguous() + ) + inp["qo_indptr"] = torch.arange(B + 1, dtype=I32) + out = torch.empty((B, 16, DV), dtype=BF16) + decode(inp, out, 1) + seqs = [b for b in range(0, B, 97) if kv_lens[b]] + sub = dict(inp) + sub["q"], sub["q_rope"] = [inp[k][seqs] for k in ("q", "q_rope")] + sub["qo_indptr"] = torch.arange(len(seqs) + 1, dtype=I32) + kvi = inp["kv_indptr"].tolist() + sub["kv_indptr"] = torch.tensor([0] + [kv_lens[b] for b in seqs]).cumsum(0).to(I32) + sub["kv_indices"] = torch.cat( + [inp["kv_indices"][kvi[b] : kvi[b + 1]] for b in seqs] + ) + assert rel_l2(out[seqs], run_torch(**sub)) < 3e-3 + cases = [] + for batch in (3, 37, 200): + inp, kv_lens = make_inputs(16, 1, batch, 659, True, seed=batch) + out = torch.empty((batch, 16, DV), dtype=BF16) + decode(inp, out, 1) + cases.append((inp, out, out.clone())) + st = torch.cuda.Stream() + graphs = [ + graph_replay(lambda i=i, o=o: decode(i, o, 1), 1, st) for i, o, _ in cases + ] + for n in range(3000): + graphs[n % 3].replay() + torch.cuda.synchronize() + assert all(torch.equal(bits(o), bits(r)) for _, o, r in cases) + inp, out, ref = cases[1] + spare = fly_v4._HDR_SPARE[out.device.index].data_ptr() + # high-priority pool: a default-priority Stream() may reuse a handle that + # already owns a workspace + s3, g = torch.cuda.Stream(priority=-1), torch.cuda.CUDAGraph() + with torch.cuda.graph(g, stream=s3): + decode(inp, out, 1) + with torch.cuda.stream(s3): + assert fly_v4._workspace(out.device).hdr.data_ptr() == spare + out.zero_() + for _ in range(100): + g.replay() + torch.cuda.synchronize() + assert torch.equal(bits(out), bits(ref)) + + +def _aiter_v4_decode(): + try: + from aiter.ops import flydsl as fly + + fly.flydsl_mla_v4_decode_supported + return fly + except (ImportError, AttributeError): + return None + + +def test_runtime_source_selection(): + """runtime prefers aiter's copy, falls back to the in-tree one; the env knobs + force one copy or turn FlyDSL off.""" + from sglang.kernels.ops.attention.dsv4.unified_kv_kernels import runtime + from sglang.srt.environ import envs + + aiter_fly = _aiter_v4_decode() + expect = { + "auto": aiter_fly or flydsl_v4nm, + "aiter": aiter_fly, + "intree": flydsl_v4nm, + } + try: + for source, fly in expect.items(): + runtime._flydsl_v4_decode.cache_clear() + with envs.SGLANG_DSV4_FLYDSL_DECODE_SOURCE.override(source): + assert runtime._flydsl_v4_decode() is fly, source + runtime._flydsl_v4_decode.cache_clear() + with envs.SGLANG_DSV4_FLYDSL_DECODE.override(False): + assert runtime._flydsl_v4_decode() is None + assert not runtime.flydsl_decode_supported(16) + finally: + runtime._flydsl_v4_decode.cache_clear() + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-v"]))