[Perf] DeepSeek-V4.1: Run the small-batch decode fusions at TP2 (#43838)

Co-authored-by: weireweire <20922698+weireweire@users.noreply.github.com>
Co-authored-by: nbommi19 <227267412+nbommi19@users.noreply.github.com>
Co-authored-by: Po-Han Huang <pohanh@nvidia.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Po-Han Huang (NVIDIA) <53919306+nvpohanh@users.noreply.github.com>
This commit is contained in:
weireweire
2026-10-19 21:17:05 -07:00
committed by GitHub
co-authored by weireweire nbommi19 Po-Han Huang Claude Opus 5.5 Po-Han Huang
parent fdd53418a0
commit 72d886efb8
9 changed files with 136 additions and 83 deletions
@@ -3,12 +3,12 @@
// Collapses three launches on the DSV4 decode/verify path (fused_rope_inplace,
// _wo_a_partial, _wo_a_reduce_quant) into one cluster-launched kernel.
//
// x [T, 2, 4096] bf16 (a strided view of a [T, 64, 512] attention output)
// W [2, 1024, 4096] bf16
// -> q [T, 2048] e4m3 + scales [8192] ue8m0, and/or y [T, 2048] bf16
// x [T, G, 4096] bf16 (a strided view of a [T, 64, 512] attention output)
// W [G, 1024, 4096] bf16
// -> q [T, G*1024] e4m3 + scales [G*4096] ue8m0, and/or y [T, G*1024] bf16
//
// Decomposition: the 2048 output columns split into 32 N-tiles of 64, one per
// cluster; K splits 8 ways across the cluster's CTAs, so each CTA owns exactly
// Decomposition: the G*1024 output columns split into G*16 N-tiles of 64, one
// per cluster; K splits 8 ways across the cluster's CTAs, so each CTA owns exactly
// one of the group's 8 heads -- and therefore exactly one 64-wide RoPE window,
// which is exactly one staged K-tile. That is what folds the rope launch in
// without a grid-wide dependency.
@@ -251,17 +251,19 @@ namespace sglang::wo_a_fused {
// Shape
// ---------------------------------------------------------------------------
/**
* \brief Compile-time shape for the TP4 WO-A decomposition.
* \brief Compile-time shape for the WO-A decomposition.
*
* \tparam kMaxTokens Token capacity; also the MMA's N axis, so a multiple of 8.
* \tparam kGroups Local o_groups: 2 at TP4, 4 at TP2.
*
* 2048 output columns split into kClusters N-tiles of kNTile, one per cluster;
* K splits kSplitK ways across the cluster's CTAs, giving each CTA exactly one
* of the group's 8 heads and therefore exactly one RoPE window.
* kGroups x 1024 output columns split into kClusters N-tiles of kNTile, one per
* cluster; K splits kSplitK ways across the cluster's CTAs, giving each CTA
* exactly one of the group's 8 heads and therefore exactly one RoPE window.
*/
template <int kMaxTokens>
template <int kMaxTokens, int kGroups = 2>
struct WoATrait {
static_assert(kMaxTokens == 16 || kMaxTokens == 32, "only the x16/x32 TMEM loads are wired up");
static_assert(kGroups == 2 || kGroups == 4, "TP4 (2 local groups) and TP2 (4) only");
static constexpr int kMMax = kMaxTokens;
static constexpr int kNTile = 64; // == 2 MXFP8 blocks
@@ -274,7 +276,8 @@ struct WoATrait {
static constexpr int kHeadStages = kHeadDim / kBlockK;
static constexpr int kHeadsPerCta = kKSlab / kHeadDim;
static constexpr int kRopeDim = 64;
static constexpr int kNOut = 2048; // 2 groups x 1024
static constexpr int kGroupN = 1024; // o_lora_rank
static constexpr int kNOut = kGroups * kGroupN;
// One UE8M0 byte per MXFP8 block, for 128 padded token rows.
static constexpr int kScaleBytes = (kNOut / 32) * 128;
static constexpr int kClusters = kNOut / kNTile;
@@ -315,7 +318,8 @@ struct WoATrait {
uint64_t tmem_ld_done;
uint32_t taddr;
};
// 2 CTAs/SM needs <= 113 KiB each; the 32-cluster wave depends on it.
// 2 CTAs/SM needs <= 113 KiB each; the single-wave TP4 grid (32 clusters) depends on it.
// TP2 launches 64 clusters / 512 CTAs, ~1.7 waves at 2 CTAs/SM, which likely bounds its WO-A gain.
static_assert(sizeof(Smem) <= 115712, "smem would drop the kernel to 1 CTA/SM");
};
@@ -360,9 +364,9 @@ __global__ void __cluster_dims__(Trait::kSplitK, 1, 1) __launch_bounds__(Trait::
const __grid_constant__ CUtensorMap x_map,
const float* __restrict__ freqs, // [S, 64] fp32, (cos0, sin0, cos1, ...)
const PosT* __restrict__ positions, // [T] int32 or int64
fp8_e4m3_t* __restrict__ q, // [T, 2048], may be null
uint8_t* __restrict__ scales_out, // [8192], may be null when q is
bf16_t* __restrict__ y, // [T, 2048], may be null
fp8_e4m3_t* __restrict__ q, // [T, kNOut], may be null
uint8_t* __restrict__ scales_out, // [kScaleBytes], may be null when q is
bf16_t* __restrict__ y, // [T, kNOut], may be null
uint32_t m) {
using namespace device;
// The trait's shape, unqualified, so the body below reads as plain constants.
@@ -377,6 +381,7 @@ __global__ void __cluster_dims__(Trait::kSplitK, 1, 1) __launch_bounds__(Trait::
constexpr int kHeadsPerCta = Trait::kHeadsPerCta;
constexpr int kRopeDim = Trait::kRopeDim;
constexpr int kNOut = Trait::kNOut;
constexpr int kGroupN = Trait::kGroupN;
constexpr int kWarps = Trait::kWarps;
constexpr int kThreads = Trait::kThreads;
constexpr int kDrainWarps = Trait::kDrainWarps;
@@ -401,8 +406,8 @@ __global__ void __cluster_dims__(Trait::kSplitK, 1, 1) __launch_bounds__(Trait::
const uint32_t lane_id = tx % kWarpThreads;
const uint32_t rank = ptx::cluster_ctarank(); // == the head this CTA owns
const uint32_t work_id = blockIdx.x / kSplitK;
const uint32_t group = work_id / (1024 / kNTile);
const uint32_t n0 = (work_id % (1024 / kNTile)) * kNTile;
const uint32_t group = work_id / (kGroupN / kNTile);
const uint32_t n0 = (work_id % (kGroupN / kNTile)) * kNTile;
const uint32_t k0 = rank * kKSlab;
// Before anything else: the descriptor fetch is serial with the first tile's
@@ -442,12 +447,12 @@ __global__ void __cluster_dims__(Trait::kSplitK, 1, 1) __launch_bounds__(Trait::
if (warp_id == TMA_W) {
// The weights are not the producer kernel's output, so this warp never waits
// on PDL and the whole 16 MiB stream overlaps the attention tail.
// on PDL and the whole weight stream overlaps the attention tail.
if (warp::elect_one_lane()) {
#pragma unroll
for (int s = 0; s < kStages; ++s) {
ptx::cp_async_bulk_tensor_2d(
ptx::to_shared(smem.w[s]), &w_map, k0 + s * kBlockK, group * 1024 + n0, &smem.w_ready[s]);
ptx::to_shared(smem.w[s]), &w_map, k0 + s * kBlockK, group * kGroupN + n0, &smem.w_ready[s]);
ptx::mbar_arrive_expect_tx(&smem.w_ready[s], kWTile);
}
}
@@ -582,7 +587,7 @@ __global__ void __cluster_dims__(Trait::kSplitK, 1, 1) __launch_bounds__(Trait::
// Only the first kEpiWarps warps own a token; inbox's middle dim is
// kTokPerRank, and t would otherwise collide with the next rank's tokens.
const auto col0 = group * 1024 + n0;
const auto col0 = group * kGroupN + n0;
const auto t = rank * kTokPerRank + warp_id;
// Unconditional, and before the guard: this is what keeps the CTA alive
// until every peer's st.async into our inbox has landed. A rank whose tokens
@@ -690,19 +695,20 @@ namespace sglang {
/**
* \brief Fused inverse-RoPE + grouped WO-A BF16 GEMM + MXFP8 quantization.
*
* \param x [T, 2, 4096] bf16 attention output; a strided view of a
* \tparam kGroups Local o_groups G.
* \param x [T, G, 4096] bf16 attention output; a strided view of a
* [T, 64, 512] buffer, so only the group and K axes need to be
* contiguous and the row stride is read off the tensor.
* \param w [2, 1024, 4096] bf16, contiguous.
* \param w [G, 1024, 4096] bf16, contiguous.
* \param freqs [S, 64] fp32 rotation table, (cos0, sin0, cos1, sin1, ...).
* \param positions [T] absolute token positions, int32 or int64.
* Exactly one output form must be supplied: either (\p q, \p scales) or \p y.
*
* \param q [T, 2048] e4m3 quantized output; pairs with \p scales.
* \param scales [8192] uint8 UE8M0 scales in FlashInfer's 128x4 swizzle.
* \param y [T, 2048] bf16, the result before quantization.
* \param q [T, G*1024] e4m3 quantized output; pairs with \p scales.
* \param scales [G*4096] uint8 UE8M0 scales in FlashInfer's 128x4 swizzle.
* \param y [T, G*1024] bf16, the result before quantization.
*/
template <int kMaxTokens>
template <int kMaxTokens, int kGroups = 2>
inline void wo_a_fused_run(
tvm::ffi::TensorView x,
tvm::ffi::TensorView w,
@@ -713,7 +719,7 @@ inline void wo_a_fused_run(
tvm::ffi::Optional<tvm::ffi::TensorView> y) {
using namespace host;
using namespace wo_a_fused; // make_map, wo_a_fused_kernel
using Trait = WoATrait<kMaxTokens>;
using Trait = WoATrait<kMaxTokens, kGroups>;
constexpr int kMMax = Trait::kMMax;
constexpr int kNTile = Trait::kNTile;
constexpr int kBlockK = Trait::kBlockK;
@@ -728,17 +734,17 @@ inline void wo_a_fused_run(
SymbolicSize T = {"num_tokens"};
// The row stride is captured rather than required: it is 32768 elements when
// the attention backend pads to 64 heads and 8192 when it does not.
// the attention backend pads to 64 heads and 512 x local heads when it does not.
SymbolicSize XS0 = {"x_row_stride"};
SymbolicDevice device_;
device_.set_options<kDLCUDA>();
TensorMatcher({T, 2, kGroupK}) //
TensorMatcher({T, kGroups, kGroupK}) //
.with_strides({XS0, kGroupK, 1})
.with_dtype<bf16_t>()
.with_device<kDLCUDA>(device_)
.verify(x);
TensorMatcher({2, kNOut / 2, kGroupK}) //
TensorMatcher({kGroups, kNOut / kGroups, kGroupK}) //
.with_dtype<bf16_t>()
.with_device<kDLCUDA>(device_)
.verify(w);
@@ -774,9 +780,10 @@ inline void wo_a_fused_run(
const int64_t x_stride0 = XS0.unwrap();
const DLDevice device = device_.unwrap();
CHECK_HOST(m >= 1 && m <= kMMax) << "wo_a_fused supports 1 <= T <= " << kMMax << ", got " << m;
CHECK_HOST(x_stride0 >= 2 * kGroupK) << "x row stride must cover both groups, got " << x_stride0;
CHECK_HOST(x_stride0 >= kGroups * kGroupK)
<< "x row stride must cover all " << kGroups << " groups, got " << x_stride0;
const auto w_map = make_map(w.data_ptr(), kGroupK, kNOut, kGroupK * 2, kBlockK, kNTile);
const auto x_map = make_map(x.data_ptr(), 2 * kGroupK, m, x_stride0 * 2, kBlockK, kMMax);
const auto x_map = make_map(x.data_ptr(), kGroups * kGroupK, m, x_stride0 * 2, kBlockK, kMMax);
const auto launch = [&](const auto* pos) {
using PosT = std::decay_t<decltype(*pos)>;
constexpr auto kernel = wo_a_fused_kernel<Trait, PosT>;
@@ -27,7 +27,7 @@ def can_use_swapab_attention(
"""The caller adds the SM100 and single-query forward-mode gates."""
return (
0 < q.shape[0] <= MAX_BATCH
and num_heads == NUM_HEADS
and num_heads in (NUM_HEADS, 2 * NUM_HEADS)
and q.dtype == torch.bfloat16
and q.shape[-1] == HEAD_DIM
and head_dim_v == HEAD_DIM
@@ -106,16 +106,20 @@ def swapab_attention(
inv_rope=None,
partial_gluon=None,
):
"""V4-layout attention on 16 heads; `extra_*` is a second slot range appended
to each request's keys, and the attention sink is folded in exactly once.
"""V4-layout attention on 16 heads, or 32 as two head groups (SM100 kernel
only); `extra_*` is a second slot range appended to each request's keys, and
the attention sink is folded in exactly once.
`partial_gluon` replaces the SM100 split-KV kernel (swapab_gluon_hip passes gfx950's)."""
b, h, d = q.shape[0], q.shape[-2], q.shape[-1]
head_groups = h // NUM_HEADS
assert h == NUM_HEADS or (h == 2 * NUM_HEADS and partial_gluon is None)
group_kwargs = {"HG": head_groups} if head_groups > 1 else {}
if partial_gluon is None:
from .decode_attention_sm100_gluon import partial_gluon
block = 64
b, h, d = q.shape[0], q.shape[-2], q.shape[-1]
assert q.ndim in (3, 4) and (q.ndim == 3 or q.shape[1] == 1)
assert 0 < b <= MAX_BATCH and h == NUM_HEADS and d == HEAD_DIM
assert 0 < b <= MAX_BATCH and d == HEAD_DIM
assert q.dtype == torch.bfloat16 and q.stride(-1) == 1
assert kv.shape[-1] == LAYOUT.bytes_per_token
assert kv.dtype in (torch.uint8, torch.float8_e4m3fn)
@@ -138,7 +142,7 @@ def swapab_attention(
maximum = torch.empty((b, nt, h), dtype=torch.float32, device=q.device)
sums = torch.empty_like(maximum)
out = torch.empty((b, h, 512), dtype=q.dtype, device=q.device)
partial_gluon[(b, nt)](
partial_gluon[(b, nt, head_groups)](
q,
kv,
extra_kv,
@@ -162,7 +166,7 @@ def swapab_attention(
NT=nt,
KT=kt,
BT=block,
H=h,
H=NUM_HEADS,
SCALE=SOFTMAX_SCALE,
KTOKENS=kv.shape[0] * kv.shape[1],
ETOKENS=extra_kv.shape[0] * extra_kv.shape[1],
@@ -172,6 +176,7 @@ def swapab_attention(
SCALE_BYTES=LAYOUT.scale_bytes,
TILE=LAYOUT.tile_size,
num_warps=4,
**group_kwargs,
)
if inv_rope is not None:
freqs, positions = inv_rope
@@ -98,9 +98,12 @@ def partial_gluon(
DATA_BYTES: gl.constexpr,
SCALE_BYTES: gl.constexpr,
TILE: gl.constexpr,
HG: gl.constexpr = 1,
):
gl.static_assert(SWAP_AB and H == 16 and (BT == 64 or BT == 128))
gl.static_assert(SWAP_AB and H == 16 and (BT == 64 or BT == 128) and HG >= 1)
HT: gl.constexpr = H * HG
b, t = gl.program_id(0), gl.program_id(1)
h0 = gl.program_id(2) * H
kv_layout: gl.constexpr = gl.BlockedLayout([1, 8], [4, 8], [4, 1], [1, 0])
n = gl.arange(0, BT, gl.SliceLayout(1, kv_layout))
if t < KT:
@@ -137,7 +140,7 @@ def partial_gluon(
)
qh = gl.arange(0, H, gl.SliceLayout(1, kv_layout))
qd = gl.arange(0, 512, gl.SliceLayout(0, kv_layout))
q = gl.load(Q + b * QS + qh[:, None] * QH + qd[None, :])
q = gl.load(Q + b * QS + (h0 + qh[:, None]) * QH + qd[None, :])
q_smem = gl.allocate_shared_memory(
gl.bfloat16,
[H, 512],
@@ -199,8 +202,8 @@ def partial_gluon(
value_hd = gl.convert_layout(value.permute((1, 0)), hd_layout)
h = gl.arange(0, H, gl.SliceLayout(1, hd_layout))
d = gl.arange(0, 512, gl.SliceLayout(0, hd_layout))
gl.store(PART + ((b * NT + t) * H + h[:, None]) * 512 + d[None, :], value_hd)
gl.store(PART + ((b * NT + t) * HT + h0 + h[:, None]) * 512 + d[None, :], value_hd)
stat_layout: gl.constexpr = gl.BlockedLayout([1], [32], [4], [0])
hs = gl.arange(0, H, stat_layout)
gl.store(MAX + (b * NT + t) * H + hs, gl.convert_layout(mx, stat_layout))
gl.store(SUM + (b * NT + t) * H + hs, gl.convert_layout(denom, stat_layout))
gl.store(MAX + (b * NT + t) * HT + h0 + hs, gl.convert_layout(mx, stat_layout))
gl.store(SUM + (b * NT + t) * HT + h0 + hs, gl.convert_layout(denom, stat_layout))
@@ -9,19 +9,21 @@ from sglang.kernels.jit.utils import cache_once, cuda_stubs_dir, load_jit
if TYPE_CHECKING:
from tvm_ffi.module import Module
GROUPS = 2
GROUPS = (2, 4)
RANK = 1024
N_OUT = GROUPS * RANK
SCALE_BYTES = 8192
MAX_M = 32
def can_use_fused_wo_a(n_local_groups: int, o_lora_rank: int) -> bool:
return n_local_groups in GROUPS and o_lora_rank == RANK
@cache_once
def _jit_module(max_tokens: int) -> Module:
def _jit_module(max_tokens: int, groups: int) -> Module:
return load_jit(
f"fused_rope_wo_a_m{max_tokens}",
f"fused_rope_wo_a_g{groups}_m{max_tokens}",
cuda_files=["deepseek_v4/wo_a_fused.cuh"],
cuda_wrappers=[("run", f"wo_a_fused_run<{max_tokens}>")],
cuda_wrappers=[("run", f"wo_a_fused_run<{max_tokens}, {groups}>")],
extra_cuda_cflags=["-O3"],
extra_ldflags=[f"-L{cuda_stubs_dir()}", "-lcuda"],
)
@@ -37,15 +39,19 @@ def fused_rope_wo_a_bf16(
) -> Sequence[torch.Tensor]:
"""Inverse RoPE + BF16 WO-A, returning BF16 or FlashInfer-swizzled MXFP8."""
num_tokens = x.shape[0]
groups = weight.shape[0]
assert num_tokens <= MAX_M
assert groups in GROUPS
n_out = groups * RANK
max_tokens = 16 if num_tokens <= 16 else 32
module = _jit_module(max_tokens)
module = _jit_module(max_tokens, groups)
if out_mxfp8:
q = torch.empty((num_tokens, N_OUT), dtype=torch.float8_e4m3fn, device=x.device)
scales = torch.empty(SCALE_BYTES, dtype=torch.uint8, device=x.device)
q = torch.empty((num_tokens, n_out), dtype=torch.float8_e4m3fn, device=x.device)
# One UE8M0 byte per 32-column block, for 128 padded token rows.
scales = torch.empty(n_out // 32 * 128, dtype=torch.uint8, device=x.device)
module.run(x, weight, freqs_cis, positions, q, scales, None)
return q, scales
else:
y = x.new_empty((num_tokens, N_OUT))
y = x.new_empty((num_tokens, n_out))
module.run(x, weight, freqs_cis, positions, None, None, y)
return (y,)
@@ -4119,7 +4119,7 @@ class DeepseekV4AttnBackend(
)
return swapab_attention(
q[..., :16, :],
q[..., : layer.tp_q_head_num, :],
swa_k_cache,
swa_page_indices,
swa_topk_lengths,
+9 -5
View File
@@ -31,7 +31,10 @@ from sglang.kernels.ops.attention.dsv4 import (
fused_rope_inplace,
)
from sglang.kernels.ops.attention.dsv4.wo_a import MAX_M as _FUSED_WO_A_MAX_TOKENS
from sglang.kernels.ops.attention.dsv4.wo_a import fused_rope_wo_a_bf16
from sglang.kernels.ops.attention.dsv4.wo_a import (
can_use_fused_wo_a,
fused_rope_wo_a_bf16,
)
from sglang.kernels.ops.attention.flash_mla_sm120 import SM120_DECODE_MAX_TOKENS
from sglang.kernels.ops.gemm.dsv4_wo_a import (
wo_a_bf16_gemv,
@@ -1365,7 +1368,7 @@ class MQALayer(MqaAttentionBase):
and not self.use_npu_arch35_mxfp8_wo_a
and self.wo_a.weight.dtype == torch.bfloat16
and self.wo_a.weight.shape == (self.n_local_groups * self.o_lora_rank, 4096)
and (self.n_local_groups, self.o_lora_rank) == (2, 1024)
and can_use_fused_wo_a(self.n_local_groups, self.o_lora_rank)
)
if _is_hip:
_hip.init_mqa_layer(self, quant_config)
@@ -3324,7 +3327,7 @@ class DeepseekV4DecoderLayerBase(nn.Module):
def _can_fuse_ffn_mhc(self) -> bool:
"""The MoE counterpart of `_can_fuse_attn_mhc`."""
return (
self.mlp.tp_size == 4
mhc.fused_mhc_tp_ok(self.mlp.tp_size)
and get_moe_a2a_backend().is_none()
and not self.mlp._shared_expert_tp1
and not should_skip_post_experts_all_reduce(is_tp_path=True)
@@ -3446,7 +3449,8 @@ class DeepseekV4CudaDecoderLayer(DeepseekV4DecoderLayerBase):
self.post_attention_layernorm, accepts_mxfp8=False, hc=self.ffn_hc
)
return (
get_parallel().tp_size == self.self_attn.attn_tp_size == 4
get_parallel().tp_size == self.self_attn.attn_tp_size
and mhc.fused_mhc_tp_ok(self.self_attn.attn_tp_size)
and boundary.norm_fusable
and mhc.can_fuse_post(self.hc_cfg)
)
@@ -4870,7 +4874,7 @@ class DeepseekV4ForCausalLM(nn.Module):
# The only deployments whose fused-collective gates can fire; registration
# must happen eagerly here, before graph capture.
if get_platform().is_blackwell and get_parallel().tp_size == 4:
if get_platform().is_blackwell and mhc.fused_mhc_tp_ok(get_parallel().tp_size):
from sglang.srt.layers.quantization.mxfp4_flashinfer_trtllm_moe import (
register_fused_all_reduce_comm,
)
@@ -45,6 +45,14 @@ _is_hip = is_hip()
_is_npu = is_npu()
_is_xpu = is_xpu()
# The fused all-reduce epilogues have world-size-templated kernels for TP2 and TP4.
_FUSED_MHC_TP_SIZES = (2, 4)
def fused_mhc_tp_ok(tp_size: int) -> bool:
return tp_size in _FUSED_MHC_TP_SIZES
# ---------------------------------------------------------------------------
# What a layer hands this module
# ---------------------------------------------------------------------------
@@ -9,7 +9,7 @@ from sglang.srt.utils import is_gfx95_supported, is_hip
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
from sglang.test.test_utils import CustomTestCase
register_cuda_ci(est_time=60, stage="base-b-kernel-unit", runner_config="4-gpu-b200")
register_cuda_ci(est_time=120, stage="base-b-kernel-unit", runner_config="4-gpu-b200")
# backend-specific: the HIP swap-AB Gluon kernel runs on gfx950.
register_amd_ci(est_time=30, stage="stage-b", runner_config="1-gpu-small-amd-mi35x")
@@ -49,6 +49,8 @@ def make_cache(page, pages=11):
def reference(q, values, ids, lengths, sink, extra=None):
heads = q.shape[-2]
def gather(vals, idx, lens):
idx = idx[:, 0]
valid = (idx >= 0) & (idx < vals.shape[0])
@@ -63,7 +65,7 @@ def reference(q, values, ids, lengths, sink, extra=None):
scores = q[:, 0].double() @ kv.transpose(1, 2) * 512**-0.5
scores.masked_fill_(~valid[:, None, :], -torch.inf)
scores = torch.cat(
(scores, sink[None, :16, None].double().expand(q.shape[0], -1, -1)), -1
(scores, sink[None, :heads, None].double().expand(q.shape[0], -1, -1)), -1
)
prob = torch.nan_to_num(torch.softmax(scores, -1)[..., :-1])
return prob @ kv
@@ -75,12 +77,12 @@ def reference(q, values, ids, lengths, sink, extra=None):
"requires Blackwell SM100/SM103 or AMD gfx950",
)
class TestSwapABAttention(CustomTestCase):
def check_case(self, b, page, nk, ne, graph=False):
def check_case(self, b, page, nk, ne, graph=False, heads=16):
torch.manual_seed(20260915 + b + nk + ne)
storage = torch.full(
(b, 1, 64, 512), float("nan"), device="cuda", dtype=torch.bfloat16
)
q = storage[:, :, :16]
q = storage[:, :, :heads]
q.copy_(torch.randn_like(q) * 0.5)
raw, kv, values = make_cache(page)
extra_raw, extra_kv, extra_values = make_cache(16)
@@ -106,7 +108,8 @@ class TestSwapABAttention(CustomTestCase):
dtype=torch.int32,
)
sink = torch.randn(64, device="cuda")
sink[0], sink[1], sink[2] = -torch.inf, torch.inf, 20.0
for h0 in range(0, heads, 16):
sink[h0], sink[h0 + 1], sink[h0 + 2] = -torch.inf, torch.inf, 20.0
originals = [x.clone() for x in (raw, extra_raw, ids, ei, q, lengths, el, sink)]
def candidate():
@@ -222,6 +225,15 @@ class TestSwapABAttention(CustomTestCase):
def test_dynamic_graph_replay(self):
self.check_case(6, 64, 192, 512, graph=True)
@unittest.skipIf(is_hip(), "the gfx950 kernel keeps the 16-head contract")
def test_tp2_head_groups(self):
"""TP2's 32 local heads run as two 16-head groups; each group must read
its own query heads and sinks and write its own output rows."""
for case in ((1, 1, 1, 0), (5, 16, 133, 0), (8, 128, 192, 1024)):
with self.subTest(case=case):
self.check_case(*case, heads=32)
self.check_case(6, 64, 192, 512, graph=True, heads=32)
if __name__ == "__main__":
unittest.main()
@@ -6,27 +6,29 @@ import torch
from sglang.kernels.ops.attention.dsv4.wo_a import fused_rope_wo_a_bf16
from sglang.test.ci.ci_register import register_cuda_ci
register_cuda_ci(est_time=30, stage="base-b-kernel-unit", runner_config="4-gpu-b200")
register_cuda_ci(est_time=60, stage="base-b-kernel-unit", runner_config="4-gpu-b200")
pytestmark = pytest.mark.skipif(
not torch.cuda.is_available() or torch.cuda.get_device_capability()[0] != 10,
reason="wo_a_fused requires SM100/SM103",
)
LOCAL_HEADS = 16
HEADS_PER_GROUP = 8
HEAD_DIM = 512
ROPE_DIM = 64
MAX_POS = 4096
N_OUT = 2048
RANK = 1024
def _inputs(tokens: int, padded: bool):
def _inputs(tokens: int, padded: bool, groups: int = 2):
device = "cuda"
torch.manual_seed(1234)
heads = 64 if padded else LOCAL_HEADS
local_heads = groups * HEADS_PER_GROUP
heads = 64 if padded else local_heads
backing = torch.randn(tokens, heads, HEAD_DIM, dtype=torch.bfloat16, device=device)
o = backing[:, :LOCAL_HEADS, :]
weight = torch.randn(2, 1024, 4096, dtype=torch.bfloat16, device=device).mul_(0.05)
o = backing[:, :local_heads, :]
weight = torch.randn(groups, RANK, 4096, dtype=torch.bfloat16, device=device)
weight.mul_(0.05)
angles = torch.outer(
torch.arange(MAX_POS, device=device, dtype=torch.float32),
1.0 / 10000.0 ** (torch.arange(32, device=device, dtype=torch.float32) / 32),
@@ -48,8 +50,10 @@ def _reference(o, weight, freqs, positions):
im = tail[..., 1::2].clone()
tail[..., 0::2] = (re * cos[:, None, :] + im * sin[:, None, :]).bfloat16().float()
tail[..., 1::2] = (im * cos[:, None, :] - re * sin[:, None, :]).bfloat16().float()
x = rotated.reshape(o.shape[0], 2, -1).double()
return torch.einsum("tgd,grd->tgr", x, weight.double()).reshape(o.shape[0], N_OUT)
groups = weight.shape[0]
x = rotated.reshape(o.shape[0], groups, -1).double()
out = torch.einsum("tgd,grd->tgr", x, weight.double())
return out.reshape(o.shape[0], groups * RANK)
def _ue8m0(amax: torch.Tensor):
@@ -71,11 +75,13 @@ def _sf_index(block: torch.Tensor, row: torch.Tensor) -> torch.Tensor:
return (block >> 2) * 512 + ((row % 32) * 4 + ((row // 32) % 4)) * 4 + (block & 3)
@pytest.mark.parametrize("groups", [2, 4])
@pytest.mark.parametrize("tokens", [1, 2, 4, 8, 13, 16, 17, 32])
@pytest.mark.parametrize("padded", [False, True])
def test_matches_reference(tokens: int, padded: bool):
o, weight, freqs, positions = _inputs(tokens, padded)
grouped = o.view(tokens, 2, -1)
def test_matches_reference(tokens: int, padded: bool, groups: int):
o, weight, freqs, positions = _inputs(tokens, padded, groups)
grouped = o.view(tokens, groups, -1)
n_out = groups * RANK
# The two output forms are mutually exclusive, so this also checks that the
# bf16 path and the quantized path agree on the same inputs.
(y,) = fused_rope_wo_a_bf16(grouped, weight, freqs, positions, out_mxfp8=False)
@@ -87,26 +93,28 @@ def test_matches_reference(tokens: int, padded: bool):
assert err < 1e-2, f"relative error {err:.3g}"
# The quantizer does: quantize(y) must match bit for bit.
yf = y.float().reshape(tokens, N_OUT // 32, 32)
yf = y.float().reshape(tokens, n_out // 32, 32)
sf, inv = _ue8m0(yf.abs().amax(-1))
want = (yf * inv[..., None]).clamp(max=448.0).to(torch.float8_e4m3fn)
torch.testing.assert_close(
q.reshape(tokens, -1, 32).float(), want.float(), rtol=0, atol=0
)
blocks = torch.arange(N_OUT // 32)
blocks = torch.arange(n_out // 32)
rows = torch.arange(tokens)[:, None]
torch.testing.assert_close(
scales.cpu()[_sf_index(blocks, rows)], sf.cpu(), rtol=0, atol=0
)
def test_scale_padding_is_zeroed():
@pytest.mark.parametrize("groups", [2, 4])
def test_scale_padding_is_zeroed(groups: int):
"""Rows >= T must be zeroed every launch: the buffer is reused by CUDA graphs."""
tokens = 4
o, weight, freqs, positions = _inputs(tokens, padded=False)
_, scales = fused_rope_wo_a_bf16(o.view(tokens, 2, -1), weight, freqs, positions)
blocks = torch.arange(N_OUT // 32)
o, weight, freqs, positions = _inputs(tokens, padded=False, groups=groups)
grouped = o.view(tokens, groups, -1)
_, scales = fused_rope_wo_a_bf16(grouped, weight, freqs, positions)
blocks = torch.arange(groups * RANK // 32)
rows = torch.arange(tokens, 128)[:, None]
assert torch.count_nonzero(scales.cpu()[_sf_index(blocks, rows)]) == 0