[Perf] DeepSeek-V4.1: Run the small-batch decode fusions at TP2 (#43838)

Co-authored-by: weireweire <20922698+weireweire@users.noreply.github.com>
Co-authored-by: nbommi19 <227267412+nbommi19@users.noreply.github.com>
Co-authored-by: Po-Han Huang <pohanh@nvidia.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Po-Han Huang (NVIDIA) <53919306+nvpohanh@users.noreply.github.com>
This commit is contained in:
weireweire
2026-10-19 21:17:05 -07:00
committed by GitHub
co-authored by weireweire nbommi19 Po-Han Huang Claude Opus 5.5 Po-Han Huang
parent fdd53418a0
commit 72d886efb8
9 changed files with 136 additions and 83 deletions
@@ -3,12 +3,12 @@
// Collapses three launches on the DSV4 decode/verify path (fused_rope_inplace,
// _wo_a_partial, _wo_a_reduce_quant) into one cluster-launched kernel.
//
// x [T, 2, 4096] bf16 (a strided view of a [T, 64, 512] attention output)
// W [2, 1024, 4096] bf16
// -> q [T, 2048] e4m3 + scales [8192] ue8m0, and/or y [T, 2048] bf16
// x [T, G, 4096] bf16 (a strided view of a [T, 64, 512] attention output)
// W [G, 1024, 4096] bf16
// -> q [T, G*1024] e4m3 + scales [G*4096] ue8m0, and/or y [T, G*1024] bf16
//
// Decomposition: the 2048 output columns split into 32 N-tiles of 64, one per
// cluster; K splits 8 ways across the cluster's CTAs, so each CTA owns exactly
// Decomposition: the G*1024 output columns split into G*16 N-tiles of 64, one
// per cluster; K splits 8 ways across the cluster's CTAs, so each CTA owns exactly
// one of the group's 8 heads -- and therefore exactly one 64-wide RoPE window,
// which is exactly one staged K-tile. That is what folds the rope launch in
// without a grid-wide dependency.
@@ -251,17 +251,19 @@ namespace sglang::wo_a_fused {
// Shape
// ---------------------------------------------------------------------------
/**
* \brief Compile-time shape for the TP4 WO-A decomposition.
* \brief Compile-time shape for the WO-A decomposition.
*
* \tparam kMaxTokens Token capacity; also the MMA's N axis, so a multiple of 8.
* \tparam kGroups Local o_groups: 2 at TP4, 4 at TP2.
*
* 2048 output columns split into kClusters N-tiles of kNTile, one per cluster;
* K splits kSplitK ways across the cluster's CTAs, giving each CTA exactly one
* of the group's 8 heads and therefore exactly one RoPE window.
* kGroups x 1024 output columns split into kClusters N-tiles of kNTile, one per
* cluster; K splits kSplitK ways across the cluster's CTAs, giving each CTA
* exactly one of the group's 8 heads and therefore exactly one RoPE window.
*/
template <int kMaxTokens>
template <int kMaxTokens, int kGroups = 2>
struct WoATrait {
static_assert(kMaxTokens == 16 || kMaxTokens == 32, "only the x16/x32 TMEM loads are wired up");
static_assert(kGroups == 2 || kGroups == 4, "TP4 (2 local groups) and TP2 (4) only");
static constexpr int kMMax = kMaxTokens;
static constexpr int kNTile = 64; // == 2 MXFP8 blocks
@@ -274,7 +276,8 @@ struct WoATrait {
static constexpr int kHeadStages = kHeadDim / kBlockK;
static constexpr int kHeadsPerCta = kKSlab / kHeadDim;
static constexpr int kRopeDim = 64;
static constexpr int kNOut = 2048; // 2 groups x 1024
static constexpr int kGroupN = 1024; // o_lora_rank
static constexpr int kNOut = kGroups * kGroupN;
// One UE8M0 byte per MXFP8 block, for 128 padded token rows.
static constexpr int kScaleBytes = (kNOut / 32) * 128;
static constexpr int kClusters = kNOut / kNTile;
@@ -315,7 +318,8 @@ struct WoATrait {
uint64_t tmem_ld_done;
uint32_t taddr;
};
// 2 CTAs/SM needs <= 113 KiB each; the 32-cluster wave depends on it.
// 2 CTAs/SM needs <= 113 KiB each; the single-wave TP4 grid (32 clusters) depends on it.
// TP2 launches 64 clusters / 512 CTAs, ~1.7 waves at 2 CTAs/SM, which likely bounds its WO-A gain.
static_assert(sizeof(Smem) <= 115712, "smem would drop the kernel to 1 CTA/SM");
};
@@ -360,9 +364,9 @@ __global__ void __cluster_dims__(Trait::kSplitK, 1, 1) __launch_bounds__(Trait::
const __grid_constant__ CUtensorMap x_map,
const float* __restrict__ freqs, // [S, 64] fp32, (cos0, sin0, cos1, ...)
const PosT* __restrict__ positions, // [T] int32 or int64
fp8_e4m3_t* __restrict__ q, // [T, 2048], may be null
uint8_t* __restrict__ scales_out, // [8192], may be null when q is
bf16_t* __restrict__ y, // [T, 2048], may be null
fp8_e4m3_t* __restrict__ q, // [T, kNOut], may be null
uint8_t* __restrict__ scales_out, // [kScaleBytes], may be null when q is
bf16_t* __restrict__ y, // [T, kNOut], may be null
uint32_t m) {
using namespace device;
// The trait's shape, unqualified, so the body below reads as plain constants.
@@ -377,6 +381,7 @@ __global__ void __cluster_dims__(Trait::kSplitK, 1, 1) __launch_bounds__(Trait::
constexpr int kHeadsPerCta = Trait::kHeadsPerCta;
constexpr int kRopeDim = Trait::kRopeDim;
constexpr int kNOut = Trait::kNOut;
constexpr int kGroupN = Trait::kGroupN;
constexpr int kWarps = Trait::kWarps;
constexpr int kThreads = Trait::kThreads;
constexpr int kDrainWarps = Trait::kDrainWarps;
@@ -401,8 +406,8 @@ __global__ void __cluster_dims__(Trait::kSplitK, 1, 1) __launch_bounds__(Trait::
const uint32_t lane_id = tx % kWarpThreads;
const uint32_t rank = ptx::cluster_ctarank(); // == the head this CTA owns
const uint32_t work_id = blockIdx.x / kSplitK;
const uint32_t group = work_id / (1024 / kNTile);
const uint32_t n0 = (work_id % (1024 / kNTile)) * kNTile;
const uint32_t group = work_id / (kGroupN / kNTile);
const uint32_t n0 = (work_id % (kGroupN / kNTile)) * kNTile;
const uint32_t k0 = rank * kKSlab;
// Before anything else: the descriptor fetch is serial with the first tile's
@@ -442,12 +447,12 @@ __global__ void __cluster_dims__(Trait::kSplitK, 1, 1) __launch_bounds__(Trait::
if (warp_id == TMA_W) {
// The weights are not the producer kernel's output, so this warp never waits
// on PDL and the whole 16 MiB stream overlaps the attention tail.
// on PDL and the whole weight stream overlaps the attention tail.
if (warp::elect_one_lane()) {
#pragma unroll
for (int s = 0; s < kStages; ++s) {
ptx::cp_async_bulk_tensor_2d(
ptx::to_shared(smem.w[s]), &w_map, k0 + s * kBlockK, group * 1024 + n0, &smem.w_ready[s]);
ptx::to_shared(smem.w[s]), &w_map, k0 + s * kBlockK, group * kGroupN + n0, &smem.w_ready[s]);
ptx::mbar_arrive_expect_tx(&smem.w_ready[s], kWTile);
}
}
@@ -582,7 +587,7 @@ __global__ void __cluster_dims__(Trait::kSplitK, 1, 1) __launch_bounds__(Trait::
// Only the first kEpiWarps warps own a token; inbox's middle dim is
// kTokPerRank, and t would otherwise collide with the next rank's tokens.
const auto col0 = group * 1024 + n0;
const auto col0 = group * kGroupN + n0;
const auto t = rank * kTokPerRank + warp_id;
// Unconditional, and before the guard: this is what keeps the CTA alive
// until every peer's st.async into our inbox has landed. A rank whose tokens
@@ -690,19 +695,20 @@ namespace sglang {
/**
* \brief Fused inverse-RoPE + grouped WO-A BF16 GEMM + MXFP8 quantization.
*
* \param x [T, 2, 4096] bf16 attention output; a strided view of a
* \tparam kGroups Local o_groups G.
* \param x [T, G, 4096] bf16 attention output; a strided view of a
* [T, 64, 512] buffer, so only the group and K axes need to be
* contiguous and the row stride is read off the tensor.
* \param w [2, 1024, 4096] bf16, contiguous.
* \param w [G, 1024, 4096] bf16, contiguous.
* \param freqs [S, 64] fp32 rotation table, (cos0, sin0, cos1, sin1, ...).
* \param positions [T] absolute token positions, int32 or int64.
* Exactly one output form must be supplied: either (\p q, \p scales) or \p y.
*
* \param q [T, 2048] e4m3 quantized output; pairs with \p scales.
* \param scales [8192] uint8 UE8M0 scales in FlashInfer's 128x4 swizzle.
* \param y [T, 2048] bf16, the result before quantization.
* \param q [T, G*1024] e4m3 quantized output; pairs with \p scales.
* \param scales [G*4096] uint8 UE8M0 scales in FlashInfer's 128x4 swizzle.
* \param y [T, G*1024] bf16, the result before quantization.
*/
template <int kMaxTokens>
template <int kMaxTokens, int kGroups = 2>
inline void wo_a_fused_run(
tvm::ffi::TensorView x,
tvm::ffi::TensorView w,
@@ -713,7 +719,7 @@ inline void wo_a_fused_run(
tvm::ffi::Optional<tvm::ffi::TensorView> y) {
using namespace host;
using namespace wo_a_fused; // make_map, wo_a_fused_kernel
using Trait = WoATrait<kMaxTokens>;
using Trait = WoATrait<kMaxTokens, kGroups>;
constexpr int kMMax = Trait::kMMax;
constexpr int kNTile = Trait::kNTile;
constexpr int kBlockK = Trait::kBlockK;
@@ -728,17 +734,17 @@ inline void wo_a_fused_run(
SymbolicSize T = {"num_tokens"};
// The row stride is captured rather than required: it is 32768 elements when
// the attention backend pads to 64 heads and 8192 when it does not.
// the attention backend pads to 64 heads and 512 x local heads when it does not.
SymbolicSize XS0 = {"x_row_stride"};
SymbolicDevice device_;
device_.set_options<kDLCUDA>();
TensorMatcher({T, 2, kGroupK}) //
TensorMatcher({T, kGroups, kGroupK}) //
.with_strides({XS0, kGroupK, 1})
.with_dtype<bf16_t>()
.with_device<kDLCUDA>(device_)
.verify(x);
TensorMatcher({2, kNOut / 2, kGroupK}) //
TensorMatcher({kGroups, kNOut / kGroups, kGroupK}) //
.with_dtype<bf16_t>()
.with_device<kDLCUDA>(device_)
.verify(w);
@@ -774,9 +780,10 @@ inline void wo_a_fused_run(
const int64_t x_stride0 = XS0.unwrap();
const DLDevice device = device_.unwrap();
CHECK_HOST(m >= 1 && m <= kMMax) << "wo_a_fused supports 1 <= T <= " << kMMax << ", got " << m;
CHECK_HOST(x_stride0 >= 2 * kGroupK) << "x row stride must cover both groups, got " << x_stride0;
CHECK_HOST(x_stride0 >= kGroups * kGroupK)
<< "x row stride must cover all " << kGroups << " groups, got " << x_stride0;
const auto w_map = make_map(w.data_ptr(), kGroupK, kNOut, kGroupK * 2, kBlockK, kNTile);
const auto x_map = make_map(x.data_ptr(), 2 * kGroupK, m, x_stride0 * 2, kBlockK, kMMax);
const auto x_map = make_map(x.data_ptr(), kGroups * kGroupK, m, x_stride0 * 2, kBlockK, kMMax);
const auto launch = [&](const auto* pos) {
using PosT = std::decay_t<decltype(*pos)>;
constexpr auto kernel = wo_a_fused_kernel<Trait, PosT>;
@@ -27,7 +27,7 @@ def can_use_swapab_attention(
"""The caller adds the SM100 and single-query forward-mode gates."""
return (
0 < q.shape[0] <= MAX_BATCH
and num_heads == NUM_HEADS
and num_heads in (NUM_HEADS, 2 * NUM_HEADS)
and q.dtype == torch.bfloat16
and q.shape[-1] == HEAD_DIM
and head_dim_v == HEAD_DIM
@@ -106,16 +106,20 @@ def swapab_attention(
inv_rope=None,
partial_gluon=None,
):
"""V4-layout attention on 16 heads; `extra_*` is a second slot range appended
to each request's keys, and the attention sink is folded in exactly once.
"""V4-layout attention on 16 heads, or 32 as two head groups (SM100 kernel
only); `extra_*` is a second slot range appended to each request's keys, and
the attention sink is folded in exactly once.
`partial_gluon` replaces the SM100 split-KV kernel (swapab_gluon_hip passes gfx950's)."""
b, h, d = q.shape[0], q.shape[-2], q.shape[-1]
head_groups = h // NUM_HEADS
assert h == NUM_HEADS or (h == 2 * NUM_HEADS and partial_gluon is None)
group_kwargs = {"HG": head_groups} if head_groups > 1 else {}
if partial_gluon is None:
from .decode_attention_sm100_gluon import partial_gluon
block = 64
b, h, d = q.shape[0], q.shape[-2], q.shape[-1]
assert q.ndim in (3, 4) and (q.ndim == 3 or q.shape[1] == 1)
assert 0 < b <= MAX_BATCH and h == NUM_HEADS and d == HEAD_DIM
assert 0 < b <= MAX_BATCH and d == HEAD_DIM
assert q.dtype == torch.bfloat16 and q.stride(-1) == 1
assert kv.shape[-1] == LAYOUT.bytes_per_token
assert kv.dtype in (torch.uint8, torch.float8_e4m3fn)
@@ -138,7 +142,7 @@ def swapab_attention(
maximum = torch.empty((b, nt, h), dtype=torch.float32, device=q.device)
sums = torch.empty_like(maximum)
out = torch.empty((b, h, 512), dtype=q.dtype, device=q.device)
partial_gluon[(b, nt)](
partial_gluon[(b, nt, head_groups)](
q,
kv,
extra_kv,
@@ -162,7 +166,7 @@ def swapab_attention(
NT=nt,
KT=kt,
BT=block,
H=h,
H=NUM_HEADS,
SCALE=SOFTMAX_SCALE,
KTOKENS=kv.shape[0] * kv.shape[1],
ETOKENS=extra_kv.shape[0] * extra_kv.shape[1],
@@ -172,6 +176,7 @@ def swapab_attention(
SCALE_BYTES=LAYOUT.scale_bytes,
TILE=LAYOUT.tile_size,
num_warps=4,
**group_kwargs,
)
if inv_rope is not None:
freqs, positions = inv_rope
@@ -98,9 +98,12 @@ def partial_gluon(
DATA_BYTES: gl.constexpr,
SCALE_BYTES: gl.constexpr,
TILE: gl.constexpr,
HG: gl.constexpr = 1,
):
gl.static_assert(SWAP_AB and H == 16 and (BT == 64 or BT == 128))
gl.static_assert(SWAP_AB and H == 16 and (BT == 64 or BT == 128) and HG >= 1)
HT: gl.constexpr = H * HG
b, t = gl.program_id(0), gl.program_id(1)
h0 = gl.program_id(2) * H
kv_layout: gl.constexpr = gl.BlockedLayout([1, 8], [4, 8], [4, 1], [1, 0])
n = gl.arange(0, BT, gl.SliceLayout(1, kv_layout))
if t < KT:
@@ -137,7 +140,7 @@ def partial_gluon(
)
qh = gl.arange(0, H, gl.SliceLayout(1, kv_layout))
qd = gl.arange(0, 512, gl.SliceLayout(0, kv_layout))
q = gl.load(Q + b * QS + qh[:, None] * QH + qd[None, :])
q = gl.load(Q + b * QS + (h0 + qh[:, None]) * QH + qd[None, :])
q_smem = gl.allocate_shared_memory(
gl.bfloat16,
[H, 512],
@@ -199,8 +202,8 @@ def partial_gluon(
value_hd = gl.convert_layout(value.permute((1, 0)), hd_layout)
h = gl.arange(0, H, gl.SliceLayout(1, hd_layout))
d = gl.arange(0, 512, gl.SliceLayout(0, hd_layout))
gl.store(PART + ((b * NT + t) * H + h[:, None]) * 512 + d[None, :], value_hd)
gl.store(PART + ((b * NT + t) * HT + h0 + h[:, None]) * 512 + d[None, :], value_hd)
stat_layout: gl.constexpr = gl.BlockedLayout([1], [32], [4], [0])
hs = gl.arange(0, H, stat_layout)
gl.store(MAX + (b * NT + t) * H + hs, gl.convert_layout(mx, stat_layout))
gl.store(SUM + (b * NT + t) * H + hs, gl.convert_layout(denom, stat_layout))
gl.store(MAX + (b * NT + t) * HT + h0 + hs, gl.convert_layout(mx, stat_layout))
gl.store(SUM + (b * NT + t) * HT + h0 + hs, gl.convert_layout(denom, stat_layout))
@@ -9,19 +9,21 @@ from sglang.kernels.jit.utils import cache_once, cuda_stubs_dir, load_jit
if TYPE_CHECKING:
from tvm_ffi.module import Module
GROUPS = 2
GROUPS = (2, 4)
RANK = 1024
N_OUT = GROUPS * RANK
SCALE_BYTES = 8192
MAX_M = 32
def can_use_fused_wo_a(n_local_groups: int, o_lora_rank: int) -> bool:
return n_local_groups in GROUPS and o_lora_rank == RANK
@cache_once
def _jit_module(max_tokens: int) -> Module:
def _jit_module(max_tokens: int, groups: int) -> Module:
return load_jit(
f"fused_rope_wo_a_m{max_tokens}",
f"fused_rope_wo_a_g{groups}_m{max_tokens}",
cuda_files=["deepseek_v4/wo_a_fused.cuh"],
cuda_wrappers=[("run", f"wo_a_fused_run<{max_tokens}>")],
cuda_wrappers=[("run", f"wo_a_fused_run<{max_tokens}, {groups}>")],
extra_cuda_cflags=["-O3"],
extra_ldflags=[f"-L{cuda_stubs_dir()}", "-lcuda"],
)
@@ -37,15 +39,19 @@ def fused_rope_wo_a_bf16(
) -> Sequence[torch.Tensor]:
"""Inverse RoPE + BF16 WO-A, returning BF16 or FlashInfer-swizzled MXFP8."""
num_tokens = x.shape[0]
groups = weight.shape[0]
assert num_tokens <= MAX_M
assert groups in GROUPS
n_out = groups * RANK
max_tokens = 16 if num_tokens <= 16 else 32
module = _jit_module(max_tokens)
module = _jit_module(max_tokens, groups)
if out_mxfp8:
q = torch.empty((num_tokens, N_OUT), dtype=torch.float8_e4m3fn, device=x.device)
scales = torch.empty(SCALE_BYTES, dtype=torch.uint8, device=x.device)
q = torch.empty((num_tokens, n_out), dtype=torch.float8_e4m3fn, device=x.device)
# One UE8M0 byte per 32-column block, for 128 padded token rows.
scales = torch.empty(n_out // 32 * 128, dtype=torch.uint8, device=x.device)
module.run(x, weight, freqs_cis, positions, q, scales, None)
return q, scales
else:
y = x.new_empty((num_tokens, N_OUT))
y = x.new_empty((num_tokens, n_out))
module.run(x, weight, freqs_cis, positions, None, None, y)
return (y,)