diff --git a/python/sglang/kernels/jit/csrc/deepseek_v4/wo_a_fused.cuh b/python/sglang/kernels/jit/csrc/deepseek_v4/wo_a_fused.cuh index 6acbca0802..0dc7fd234b 100644 --- a/python/sglang/kernels/jit/csrc/deepseek_v4/wo_a_fused.cuh +++ b/python/sglang/kernels/jit/csrc/deepseek_v4/wo_a_fused.cuh @@ -3,12 +3,12 @@ // Collapses three launches on the DSV4 decode/verify path (fused_rope_inplace, // _wo_a_partial, _wo_a_reduce_quant) into one cluster-launched kernel. // -// x [T, 2, 4096] bf16 (a strided view of a [T, 64, 512] attention output) -// W [2, 1024, 4096] bf16 -// -> q [T, 2048] e4m3 + scales [8192] ue8m0, and/or y [T, 2048] bf16 +// x [T, G, 4096] bf16 (a strided view of a [T, 64, 512] attention output) +// W [G, 1024, 4096] bf16 +// -> q [T, G*1024] e4m3 + scales [G*4096] ue8m0, and/or y [T, G*1024] bf16 // -// Decomposition: the 2048 output columns split into 32 N-tiles of 64, one per -// cluster; K splits 8 ways across the cluster's CTAs, so each CTA owns exactly +// Decomposition: the G*1024 output columns split into G*16 N-tiles of 64, one +// per cluster; K splits 8 ways across the cluster's CTAs, so each CTA owns exactly // one of the group's 8 heads -- and therefore exactly one 64-wide RoPE window, // which is exactly one staged K-tile. That is what folds the rope launch in // without a grid-wide dependency. @@ -251,17 +251,19 @@ namespace sglang::wo_a_fused { // Shape // --------------------------------------------------------------------------- /** - * \brief Compile-time shape for the TP4 WO-A decomposition. + * \brief Compile-time shape for the WO-A decomposition. * * \tparam kMaxTokens Token capacity; also the MMA's N axis, so a multiple of 8. + * \tparam kGroups Local o_groups: 2 at TP4, 4 at TP2. * - * 2048 output columns split into kClusters N-tiles of kNTile, one per cluster; - * K splits kSplitK ways across the cluster's CTAs, giving each CTA exactly one - * of the group's 8 heads and therefore exactly one RoPE window. + * kGroups x 1024 output columns split into kClusters N-tiles of kNTile, one per + * cluster; K splits kSplitK ways across the cluster's CTAs, giving each CTA + * exactly one of the group's 8 heads and therefore exactly one RoPE window. */ -template +template struct WoATrait { static_assert(kMaxTokens == 16 || kMaxTokens == 32, "only the x16/x32 TMEM loads are wired up"); + static_assert(kGroups == 2 || kGroups == 4, "TP4 (2 local groups) and TP2 (4) only"); static constexpr int kMMax = kMaxTokens; static constexpr int kNTile = 64; // == 2 MXFP8 blocks @@ -274,7 +276,8 @@ struct WoATrait { static constexpr int kHeadStages = kHeadDim / kBlockK; static constexpr int kHeadsPerCta = kKSlab / kHeadDim; static constexpr int kRopeDim = 64; - static constexpr int kNOut = 2048; // 2 groups x 1024 + static constexpr int kGroupN = 1024; // o_lora_rank + static constexpr int kNOut = kGroups * kGroupN; // One UE8M0 byte per MXFP8 block, for 128 padded token rows. static constexpr int kScaleBytes = (kNOut / 32) * 128; static constexpr int kClusters = kNOut / kNTile; @@ -315,7 +318,8 @@ struct WoATrait { uint64_t tmem_ld_done; uint32_t taddr; }; - // 2 CTAs/SM needs <= 113 KiB each; the 32-cluster wave depends on it. + // 2 CTAs/SM needs <= 113 KiB each; the single-wave TP4 grid (32 clusters) depends on it. + // TP2 launches 64 clusters / 512 CTAs, ~1.7 waves at 2 CTAs/SM, which likely bounds its WO-A gain. static_assert(sizeof(Smem) <= 115712, "smem would drop the kernel to 1 CTA/SM"); }; @@ -360,9 +364,9 @@ __global__ void __cluster_dims__(Trait::kSplitK, 1, 1) __launch_bounds__(Trait:: const __grid_constant__ CUtensorMap x_map, const float* __restrict__ freqs, // [S, 64] fp32, (cos0, sin0, cos1, ...) const PosT* __restrict__ positions, // [T] int32 or int64 - fp8_e4m3_t* __restrict__ q, // [T, 2048], may be null - uint8_t* __restrict__ scales_out, // [8192], may be null when q is - bf16_t* __restrict__ y, // [T, 2048], may be null + fp8_e4m3_t* __restrict__ q, // [T, kNOut], may be null + uint8_t* __restrict__ scales_out, // [kScaleBytes], may be null when q is + bf16_t* __restrict__ y, // [T, kNOut], may be null uint32_t m) { using namespace device; // The trait's shape, unqualified, so the body below reads as plain constants. @@ -377,6 +381,7 @@ __global__ void __cluster_dims__(Trait::kSplitK, 1, 1) __launch_bounds__(Trait:: constexpr int kHeadsPerCta = Trait::kHeadsPerCta; constexpr int kRopeDim = Trait::kRopeDim; constexpr int kNOut = Trait::kNOut; + constexpr int kGroupN = Trait::kGroupN; constexpr int kWarps = Trait::kWarps; constexpr int kThreads = Trait::kThreads; constexpr int kDrainWarps = Trait::kDrainWarps; @@ -401,8 +406,8 @@ __global__ void __cluster_dims__(Trait::kSplitK, 1, 1) __launch_bounds__(Trait:: const uint32_t lane_id = tx % kWarpThreads; const uint32_t rank = ptx::cluster_ctarank(); // == the head this CTA owns const uint32_t work_id = blockIdx.x / kSplitK; - const uint32_t group = work_id / (1024 / kNTile); - const uint32_t n0 = (work_id % (1024 / kNTile)) * kNTile; + const uint32_t group = work_id / (kGroupN / kNTile); + const uint32_t n0 = (work_id % (kGroupN / kNTile)) * kNTile; const uint32_t k0 = rank * kKSlab; // Before anything else: the descriptor fetch is serial with the first tile's @@ -442,12 +447,12 @@ __global__ void __cluster_dims__(Trait::kSplitK, 1, 1) __launch_bounds__(Trait:: if (warp_id == TMA_W) { // The weights are not the producer kernel's output, so this warp never waits - // on PDL and the whole 16 MiB stream overlaps the attention tail. + // on PDL and the whole weight stream overlaps the attention tail. if (warp::elect_one_lane()) { #pragma unroll for (int s = 0; s < kStages; ++s) { ptx::cp_async_bulk_tensor_2d( - ptx::to_shared(smem.w[s]), &w_map, k0 + s * kBlockK, group * 1024 + n0, &smem.w_ready[s]); + ptx::to_shared(smem.w[s]), &w_map, k0 + s * kBlockK, group * kGroupN + n0, &smem.w_ready[s]); ptx::mbar_arrive_expect_tx(&smem.w_ready[s], kWTile); } } @@ -582,7 +587,7 @@ __global__ void __cluster_dims__(Trait::kSplitK, 1, 1) __launch_bounds__(Trait:: // Only the first kEpiWarps warps own a token; inbox's middle dim is // kTokPerRank, and t would otherwise collide with the next rank's tokens. - const auto col0 = group * 1024 + n0; + const auto col0 = group * kGroupN + n0; const auto t = rank * kTokPerRank + warp_id; // Unconditional, and before the guard: this is what keeps the CTA alive // until every peer's st.async into our inbox has landed. A rank whose tokens @@ -690,19 +695,20 @@ namespace sglang { /** * \brief Fused inverse-RoPE + grouped WO-A BF16 GEMM + MXFP8 quantization. * - * \param x [T, 2, 4096] bf16 attention output; a strided view of a + * \tparam kGroups Local o_groups G. + * \param x [T, G, 4096] bf16 attention output; a strided view of a * [T, 64, 512] buffer, so only the group and K axes need to be * contiguous and the row stride is read off the tensor. - * \param w [2, 1024, 4096] bf16, contiguous. + * \param w [G, 1024, 4096] bf16, contiguous. * \param freqs [S, 64] fp32 rotation table, (cos0, sin0, cos1, sin1, ...). * \param positions [T] absolute token positions, int32 or int64. * Exactly one output form must be supplied: either (\p q, \p scales) or \p y. * - * \param q [T, 2048] e4m3 quantized output; pairs with \p scales. - * \param scales [8192] uint8 UE8M0 scales in FlashInfer's 128x4 swizzle. - * \param y [T, 2048] bf16, the result before quantization. + * \param q [T, G*1024] e4m3 quantized output; pairs with \p scales. + * \param scales [G*4096] uint8 UE8M0 scales in FlashInfer's 128x4 swizzle. + * \param y [T, G*1024] bf16, the result before quantization. */ -template +template inline void wo_a_fused_run( tvm::ffi::TensorView x, tvm::ffi::TensorView w, @@ -713,7 +719,7 @@ inline void wo_a_fused_run( tvm::ffi::Optional y) { using namespace host; using namespace wo_a_fused; // make_map, wo_a_fused_kernel - using Trait = WoATrait; + using Trait = WoATrait; constexpr int kMMax = Trait::kMMax; constexpr int kNTile = Trait::kNTile; constexpr int kBlockK = Trait::kBlockK; @@ -728,17 +734,17 @@ inline void wo_a_fused_run( SymbolicSize T = {"num_tokens"}; // The row stride is captured rather than required: it is 32768 elements when - // the attention backend pads to 64 heads and 8192 when it does not. + // the attention backend pads to 64 heads and 512 x local heads when it does not. SymbolicSize XS0 = {"x_row_stride"}; SymbolicDevice device_; device_.set_options(); - TensorMatcher({T, 2, kGroupK}) // + TensorMatcher({T, kGroups, kGroupK}) // .with_strides({XS0, kGroupK, 1}) .with_dtype() .with_device(device_) .verify(x); - TensorMatcher({2, kNOut / 2, kGroupK}) // + TensorMatcher({kGroups, kNOut / kGroups, kGroupK}) // .with_dtype() .with_device(device_) .verify(w); @@ -774,9 +780,10 @@ inline void wo_a_fused_run( const int64_t x_stride0 = XS0.unwrap(); const DLDevice device = device_.unwrap(); CHECK_HOST(m >= 1 && m <= kMMax) << "wo_a_fused supports 1 <= T <= " << kMMax << ", got " << m; - CHECK_HOST(x_stride0 >= 2 * kGroupK) << "x row stride must cover both groups, got " << x_stride0; + CHECK_HOST(x_stride0 >= kGroups * kGroupK) + << "x row stride must cover all " << kGroups << " groups, got " << x_stride0; const auto w_map = make_map(w.data_ptr(), kGroupK, kNOut, kGroupK * 2, kBlockK, kNTile); - const auto x_map = make_map(x.data_ptr(), 2 * kGroupK, m, x_stride0 * 2, kBlockK, kMMax); + const auto x_map = make_map(x.data_ptr(), kGroups * kGroupK, m, x_stride0 * 2, kBlockK, kMMax); const auto launch = [&](const auto* pos) { using PosT = std::decay_t; constexpr auto kernel = wo_a_fused_kernel; diff --git a/python/sglang/kernels/ops/attention/dsv4/decode_attention_sm100.py b/python/sglang/kernels/ops/attention/dsv4/decode_attention_sm100.py index f0e57b9496..c8e85fa119 100644 --- a/python/sglang/kernels/ops/attention/dsv4/decode_attention_sm100.py +++ b/python/sglang/kernels/ops/attention/dsv4/decode_attention_sm100.py @@ -27,7 +27,7 @@ def can_use_swapab_attention( """The caller adds the SM100 and single-query forward-mode gates.""" return ( 0 < q.shape[0] <= MAX_BATCH - and num_heads == NUM_HEADS + and num_heads in (NUM_HEADS, 2 * NUM_HEADS) and q.dtype == torch.bfloat16 and q.shape[-1] == HEAD_DIM and head_dim_v == HEAD_DIM @@ -106,16 +106,20 @@ def swapab_attention( inv_rope=None, partial_gluon=None, ): - """V4-layout attention on 16 heads; `extra_*` is a second slot range appended - to each request's keys, and the attention sink is folded in exactly once. + """V4-layout attention on 16 heads, or 32 as two head groups (SM100 kernel + only); `extra_*` is a second slot range appended to each request's keys, and + the attention sink is folded in exactly once. `partial_gluon` replaces the SM100 split-KV kernel (swapab_gluon_hip passes gfx950's).""" + b, h, d = q.shape[0], q.shape[-2], q.shape[-1] + head_groups = h // NUM_HEADS + assert h == NUM_HEADS or (h == 2 * NUM_HEADS and partial_gluon is None) + group_kwargs = {"HG": head_groups} if head_groups > 1 else {} if partial_gluon is None: from .decode_attention_sm100_gluon import partial_gluon block = 64 - b, h, d = q.shape[0], q.shape[-2], q.shape[-1] assert q.ndim in (3, 4) and (q.ndim == 3 or q.shape[1] == 1) - assert 0 < b <= MAX_BATCH and h == NUM_HEADS and d == HEAD_DIM + assert 0 < b <= MAX_BATCH and d == HEAD_DIM assert q.dtype == torch.bfloat16 and q.stride(-1) == 1 assert kv.shape[-1] == LAYOUT.bytes_per_token assert kv.dtype in (torch.uint8, torch.float8_e4m3fn) @@ -138,7 +142,7 @@ def swapab_attention( maximum = torch.empty((b, nt, h), dtype=torch.float32, device=q.device) sums = torch.empty_like(maximum) out = torch.empty((b, h, 512), dtype=q.dtype, device=q.device) - partial_gluon[(b, nt)]( + partial_gluon[(b, nt, head_groups)]( q, kv, extra_kv, @@ -162,7 +166,7 @@ def swapab_attention( NT=nt, KT=kt, BT=block, - H=h, + H=NUM_HEADS, SCALE=SOFTMAX_SCALE, KTOKENS=kv.shape[0] * kv.shape[1], ETOKENS=extra_kv.shape[0] * extra_kv.shape[1], @@ -172,6 +176,7 @@ def swapab_attention( SCALE_BYTES=LAYOUT.scale_bytes, TILE=LAYOUT.tile_size, num_warps=4, + **group_kwargs, ) if inv_rope is not None: freqs, positions = inv_rope diff --git a/python/sglang/kernels/ops/attention/dsv4/decode_attention_sm100_gluon.py b/python/sglang/kernels/ops/attention/dsv4/decode_attention_sm100_gluon.py index ec34f68e12..432d0a4dd7 100644 --- a/python/sglang/kernels/ops/attention/dsv4/decode_attention_sm100_gluon.py +++ b/python/sglang/kernels/ops/attention/dsv4/decode_attention_sm100_gluon.py @@ -98,9 +98,12 @@ def partial_gluon( DATA_BYTES: gl.constexpr, SCALE_BYTES: gl.constexpr, TILE: gl.constexpr, + HG: gl.constexpr = 1, ): - gl.static_assert(SWAP_AB and H == 16 and (BT == 64 or BT == 128)) + gl.static_assert(SWAP_AB and H == 16 and (BT == 64 or BT == 128) and HG >= 1) + HT: gl.constexpr = H * HG b, t = gl.program_id(0), gl.program_id(1) + h0 = gl.program_id(2) * H kv_layout: gl.constexpr = gl.BlockedLayout([1, 8], [4, 8], [4, 1], [1, 0]) n = gl.arange(0, BT, gl.SliceLayout(1, kv_layout)) if t < KT: @@ -137,7 +140,7 @@ def partial_gluon( ) qh = gl.arange(0, H, gl.SliceLayout(1, kv_layout)) qd = gl.arange(0, 512, gl.SliceLayout(0, kv_layout)) - q = gl.load(Q + b * QS + qh[:, None] * QH + qd[None, :]) + q = gl.load(Q + b * QS + (h0 + qh[:, None]) * QH + qd[None, :]) q_smem = gl.allocate_shared_memory( gl.bfloat16, [H, 512], @@ -199,8 +202,8 @@ def partial_gluon( value_hd = gl.convert_layout(value.permute((1, 0)), hd_layout) h = gl.arange(0, H, gl.SliceLayout(1, hd_layout)) d = gl.arange(0, 512, gl.SliceLayout(0, hd_layout)) - gl.store(PART + ((b * NT + t) * H + h[:, None]) * 512 + d[None, :], value_hd) + gl.store(PART + ((b * NT + t) * HT + h0 + h[:, None]) * 512 + d[None, :], value_hd) stat_layout: gl.constexpr = gl.BlockedLayout([1], [32], [4], [0]) hs = gl.arange(0, H, stat_layout) - gl.store(MAX + (b * NT + t) * H + hs, gl.convert_layout(mx, stat_layout)) - gl.store(SUM + (b * NT + t) * H + hs, gl.convert_layout(denom, stat_layout)) + gl.store(MAX + (b * NT + t) * HT + h0 + hs, gl.convert_layout(mx, stat_layout)) + gl.store(SUM + (b * NT + t) * HT + h0 + hs, gl.convert_layout(denom, stat_layout)) diff --git a/python/sglang/kernels/ops/attention/dsv4/wo_a.py b/python/sglang/kernels/ops/attention/dsv4/wo_a.py index 08df08123f..69bcba87e7 100644 --- a/python/sglang/kernels/ops/attention/dsv4/wo_a.py +++ b/python/sglang/kernels/ops/attention/dsv4/wo_a.py @@ -9,19 +9,21 @@ from sglang.kernels.jit.utils import cache_once, cuda_stubs_dir, load_jit if TYPE_CHECKING: from tvm_ffi.module import Module -GROUPS = 2 +GROUPS = (2, 4) RANK = 1024 -N_OUT = GROUPS * RANK -SCALE_BYTES = 8192 MAX_M = 32 +def can_use_fused_wo_a(n_local_groups: int, o_lora_rank: int) -> bool: + return n_local_groups in GROUPS and o_lora_rank == RANK + + @cache_once -def _jit_module(max_tokens: int) -> Module: +def _jit_module(max_tokens: int, groups: int) -> Module: return load_jit( - f"fused_rope_wo_a_m{max_tokens}", + f"fused_rope_wo_a_g{groups}_m{max_tokens}", cuda_files=["deepseek_v4/wo_a_fused.cuh"], - cuda_wrappers=[("run", f"wo_a_fused_run<{max_tokens}>")], + cuda_wrappers=[("run", f"wo_a_fused_run<{max_tokens}, {groups}>")], extra_cuda_cflags=["-O3"], extra_ldflags=[f"-L{cuda_stubs_dir()}", "-lcuda"], ) @@ -37,15 +39,19 @@ def fused_rope_wo_a_bf16( ) -> Sequence[torch.Tensor]: """Inverse RoPE + BF16 WO-A, returning BF16 or FlashInfer-swizzled MXFP8.""" num_tokens = x.shape[0] + groups = weight.shape[0] assert num_tokens <= MAX_M + assert groups in GROUPS + n_out = groups * RANK max_tokens = 16 if num_tokens <= 16 else 32 - module = _jit_module(max_tokens) + module = _jit_module(max_tokens, groups) if out_mxfp8: - q = torch.empty((num_tokens, N_OUT), dtype=torch.float8_e4m3fn, device=x.device) - scales = torch.empty(SCALE_BYTES, dtype=torch.uint8, device=x.device) + q = torch.empty((num_tokens, n_out), dtype=torch.float8_e4m3fn, device=x.device) + # One UE8M0 byte per 32-column block, for 128 padded token rows. + scales = torch.empty(n_out // 32 * 128, dtype=torch.uint8, device=x.device) module.run(x, weight, freqs_cis, positions, q, scales, None) return q, scales else: - y = x.new_empty((num_tokens, N_OUT)) + y = x.new_empty((num_tokens, n_out)) module.run(x, weight, freqs_cis, positions, None, None, y) return (y,) diff --git a/python/sglang/srt/layers/attention/deepseek_v4_backend.py b/python/sglang/srt/layers/attention/deepseek_v4_backend.py index 4ebb31c221..0053836cbe 100644 --- a/python/sglang/srt/layers/attention/deepseek_v4_backend.py +++ b/python/sglang/srt/layers/attention/deepseek_v4_backend.py @@ -4119,7 +4119,7 @@ class DeepseekV4AttnBackend( ) return swapab_attention( - q[..., :16, :], + q[..., : layer.tp_q_head_num, :], swa_k_cache, swa_page_indices, swa_topk_lengths, diff --git a/python/sglang/srt/models/deepseek_v4.py b/python/sglang/srt/models/deepseek_v4.py index 362ea60bf0..dc07f75a91 100644 --- a/python/sglang/srt/models/deepseek_v4.py +++ b/python/sglang/srt/models/deepseek_v4.py @@ -31,7 +31,10 @@ from sglang.kernels.ops.attention.dsv4 import ( fused_rope_inplace, ) from sglang.kernels.ops.attention.dsv4.wo_a import MAX_M as _FUSED_WO_A_MAX_TOKENS -from sglang.kernels.ops.attention.dsv4.wo_a import fused_rope_wo_a_bf16 +from sglang.kernels.ops.attention.dsv4.wo_a import ( + can_use_fused_wo_a, + fused_rope_wo_a_bf16, +) from sglang.kernels.ops.attention.flash_mla_sm120 import SM120_DECODE_MAX_TOKENS from sglang.kernels.ops.gemm.dsv4_wo_a import ( wo_a_bf16_gemv, @@ -1365,7 +1368,7 @@ class MQALayer(MqaAttentionBase): and not self.use_npu_arch35_mxfp8_wo_a and self.wo_a.weight.dtype == torch.bfloat16 and self.wo_a.weight.shape == (self.n_local_groups * self.o_lora_rank, 4096) - and (self.n_local_groups, self.o_lora_rank) == (2, 1024) + and can_use_fused_wo_a(self.n_local_groups, self.o_lora_rank) ) if _is_hip: _hip.init_mqa_layer(self, quant_config) @@ -3324,7 +3327,7 @@ class DeepseekV4DecoderLayerBase(nn.Module): def _can_fuse_ffn_mhc(self) -> bool: """The MoE counterpart of `_can_fuse_attn_mhc`.""" return ( - self.mlp.tp_size == 4 + mhc.fused_mhc_tp_ok(self.mlp.tp_size) and get_moe_a2a_backend().is_none() and not self.mlp._shared_expert_tp1 and not should_skip_post_experts_all_reduce(is_tp_path=True) @@ -3446,7 +3449,8 @@ class DeepseekV4CudaDecoderLayer(DeepseekV4DecoderLayerBase): self.post_attention_layernorm, accepts_mxfp8=False, hc=self.ffn_hc ) return ( - get_parallel().tp_size == self.self_attn.attn_tp_size == 4 + get_parallel().tp_size == self.self_attn.attn_tp_size + and mhc.fused_mhc_tp_ok(self.self_attn.attn_tp_size) and boundary.norm_fusable and mhc.can_fuse_post(self.hc_cfg) ) @@ -4870,7 +4874,7 @@ class DeepseekV4ForCausalLM(nn.Module): # The only deployments whose fused-collective gates can fire; registration # must happen eagerly here, before graph capture. - if get_platform().is_blackwell and get_parallel().tp_size == 4: + if get_platform().is_blackwell and mhc.fused_mhc_tp_ok(get_parallel().tp_size): from sglang.srt.layers.quantization.mxfp4_flashinfer_trtllm_moe import ( register_fused_all_reduce_comm, ) diff --git a/python/sglang/srt/models/deepseek_v4_mhc.py b/python/sglang/srt/models/deepseek_v4_mhc.py index a4da0b1e44..87887186b9 100644 --- a/python/sglang/srt/models/deepseek_v4_mhc.py +++ b/python/sglang/srt/models/deepseek_v4_mhc.py @@ -45,6 +45,14 @@ _is_hip = is_hip() _is_npu = is_npu() _is_xpu = is_xpu() +# The fused all-reduce epilogues have world-size-templated kernels for TP2 and TP4. +_FUSED_MHC_TP_SIZES = (2, 4) + + +def fused_mhc_tp_ok(tp_size: int) -> bool: + return tp_size in _FUSED_MHC_TP_SIZES + + # --------------------------------------------------------------------------- # What a layer hands this module # --------------------------------------------------------------------------- diff --git a/test/registered/kernels/ops/attention/dsv4/test_swapab_attention.py b/test/registered/kernels/ops/attention/dsv4/test_swapab_attention.py index 960d1a7984..acc7c22894 100644 --- a/test/registered/kernels/ops/attention/dsv4/test_swapab_attention.py +++ b/test/registered/kernels/ops/attention/dsv4/test_swapab_attention.py @@ -9,7 +9,7 @@ from sglang.srt.utils import is_gfx95_supported, is_hip from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci from sglang.test.test_utils import CustomTestCase -register_cuda_ci(est_time=60, stage="base-b-kernel-unit", runner_config="4-gpu-b200") +register_cuda_ci(est_time=120, stage="base-b-kernel-unit", runner_config="4-gpu-b200") # backend-specific: the HIP swap-AB Gluon kernel runs on gfx950. register_amd_ci(est_time=30, stage="stage-b", runner_config="1-gpu-small-amd-mi35x") @@ -49,6 +49,8 @@ def make_cache(page, pages=11): def reference(q, values, ids, lengths, sink, extra=None): + heads = q.shape[-2] + def gather(vals, idx, lens): idx = idx[:, 0] valid = (idx >= 0) & (idx < vals.shape[0]) @@ -63,7 +65,7 @@ def reference(q, values, ids, lengths, sink, extra=None): scores = q[:, 0].double() @ kv.transpose(1, 2) * 512**-0.5 scores.masked_fill_(~valid[:, None, :], -torch.inf) scores = torch.cat( - (scores, sink[None, :16, None].double().expand(q.shape[0], -1, -1)), -1 + (scores, sink[None, :heads, None].double().expand(q.shape[0], -1, -1)), -1 ) prob = torch.nan_to_num(torch.softmax(scores, -1)[..., :-1]) return prob @ kv @@ -75,12 +77,12 @@ def reference(q, values, ids, lengths, sink, extra=None): "requires Blackwell SM100/SM103 or AMD gfx950", ) class TestSwapABAttention(CustomTestCase): - def check_case(self, b, page, nk, ne, graph=False): + def check_case(self, b, page, nk, ne, graph=False, heads=16): torch.manual_seed(20260915 + b + nk + ne) storage = torch.full( (b, 1, 64, 512), float("nan"), device="cuda", dtype=torch.bfloat16 ) - q = storage[:, :, :16] + q = storage[:, :, :heads] q.copy_(torch.randn_like(q) * 0.5) raw, kv, values = make_cache(page) extra_raw, extra_kv, extra_values = make_cache(16) @@ -106,7 +108,8 @@ class TestSwapABAttention(CustomTestCase): dtype=torch.int32, ) sink = torch.randn(64, device="cuda") - sink[0], sink[1], sink[2] = -torch.inf, torch.inf, 20.0 + for h0 in range(0, heads, 16): + sink[h0], sink[h0 + 1], sink[h0 + 2] = -torch.inf, torch.inf, 20.0 originals = [x.clone() for x in (raw, extra_raw, ids, ei, q, lengths, el, sink)] def candidate(): @@ -222,6 +225,15 @@ class TestSwapABAttention(CustomTestCase): def test_dynamic_graph_replay(self): self.check_case(6, 64, 192, 512, graph=True) + @unittest.skipIf(is_hip(), "the gfx950 kernel keeps the 16-head contract") + def test_tp2_head_groups(self): + """TP2's 32 local heads run as two 16-head groups; each group must read + its own query heads and sinks and write its own output rows.""" + for case in ((1, 1, 1, 0), (5, 16, 133, 0), (8, 128, 192, 1024)): + with self.subTest(case=case): + self.check_case(*case, heads=32) + self.check_case(6, 64, 192, 512, graph=True, heads=32) + if __name__ == "__main__": unittest.main() diff --git a/test/registered/kernels/ops/attention/test_wo_a_fused.py b/test/registered/kernels/ops/attention/test_wo_a_fused.py index 96d9e462d3..6fe9f882c0 100644 --- a/test/registered/kernels/ops/attention/test_wo_a_fused.py +++ b/test/registered/kernels/ops/attention/test_wo_a_fused.py @@ -6,27 +6,29 @@ import torch from sglang.kernels.ops.attention.dsv4.wo_a import fused_rope_wo_a_bf16 from sglang.test.ci.ci_register import register_cuda_ci -register_cuda_ci(est_time=30, stage="base-b-kernel-unit", runner_config="4-gpu-b200") +register_cuda_ci(est_time=60, stage="base-b-kernel-unit", runner_config="4-gpu-b200") pytestmark = pytest.mark.skipif( not torch.cuda.is_available() or torch.cuda.get_device_capability()[0] != 10, reason="wo_a_fused requires SM100/SM103", ) -LOCAL_HEADS = 16 +HEADS_PER_GROUP = 8 HEAD_DIM = 512 ROPE_DIM = 64 MAX_POS = 4096 -N_OUT = 2048 +RANK = 1024 -def _inputs(tokens: int, padded: bool): +def _inputs(tokens: int, padded: bool, groups: int = 2): device = "cuda" torch.manual_seed(1234) - heads = 64 if padded else LOCAL_HEADS + local_heads = groups * HEADS_PER_GROUP + heads = 64 if padded else local_heads backing = torch.randn(tokens, heads, HEAD_DIM, dtype=torch.bfloat16, device=device) - o = backing[:, :LOCAL_HEADS, :] - weight = torch.randn(2, 1024, 4096, dtype=torch.bfloat16, device=device).mul_(0.05) + o = backing[:, :local_heads, :] + weight = torch.randn(groups, RANK, 4096, dtype=torch.bfloat16, device=device) + weight.mul_(0.05) angles = torch.outer( torch.arange(MAX_POS, device=device, dtype=torch.float32), 1.0 / 10000.0 ** (torch.arange(32, device=device, dtype=torch.float32) / 32), @@ -48,8 +50,10 @@ def _reference(o, weight, freqs, positions): im = tail[..., 1::2].clone() tail[..., 0::2] = (re * cos[:, None, :] + im * sin[:, None, :]).bfloat16().float() tail[..., 1::2] = (im * cos[:, None, :] - re * sin[:, None, :]).bfloat16().float() - x = rotated.reshape(o.shape[0], 2, -1).double() - return torch.einsum("tgd,grd->tgr", x, weight.double()).reshape(o.shape[0], N_OUT) + groups = weight.shape[0] + x = rotated.reshape(o.shape[0], groups, -1).double() + out = torch.einsum("tgd,grd->tgr", x, weight.double()) + return out.reshape(o.shape[0], groups * RANK) def _ue8m0(amax: torch.Tensor): @@ -71,11 +75,13 @@ def _sf_index(block: torch.Tensor, row: torch.Tensor) -> torch.Tensor: return (block >> 2) * 512 + ((row % 32) * 4 + ((row // 32) % 4)) * 4 + (block & 3) +@pytest.mark.parametrize("groups", [2, 4]) @pytest.mark.parametrize("tokens", [1, 2, 4, 8, 13, 16, 17, 32]) @pytest.mark.parametrize("padded", [False, True]) -def test_matches_reference(tokens: int, padded: bool): - o, weight, freqs, positions = _inputs(tokens, padded) - grouped = o.view(tokens, 2, -1) +def test_matches_reference(tokens: int, padded: bool, groups: int): + o, weight, freqs, positions = _inputs(tokens, padded, groups) + grouped = o.view(tokens, groups, -1) + n_out = groups * RANK # The two output forms are mutually exclusive, so this also checks that the # bf16 path and the quantized path agree on the same inputs. (y,) = fused_rope_wo_a_bf16(grouped, weight, freqs, positions, out_mxfp8=False) @@ -87,26 +93,28 @@ def test_matches_reference(tokens: int, padded: bool): assert err < 1e-2, f"relative error {err:.3g}" # The quantizer does: quantize(y) must match bit for bit. - yf = y.float().reshape(tokens, N_OUT // 32, 32) + yf = y.float().reshape(tokens, n_out // 32, 32) sf, inv = _ue8m0(yf.abs().amax(-1)) want = (yf * inv[..., None]).clamp(max=448.0).to(torch.float8_e4m3fn) torch.testing.assert_close( q.reshape(tokens, -1, 32).float(), want.float(), rtol=0, atol=0 ) - blocks = torch.arange(N_OUT // 32) + blocks = torch.arange(n_out // 32) rows = torch.arange(tokens)[:, None] torch.testing.assert_close( scales.cpu()[_sf_index(blocks, rows)], sf.cpu(), rtol=0, atol=0 ) -def test_scale_padding_is_zeroed(): +@pytest.mark.parametrize("groups", [2, 4]) +def test_scale_padding_is_zeroed(groups: int): """Rows >= T must be zeroed every launch: the buffer is reused by CUDA graphs.""" tokens = 4 - o, weight, freqs, positions = _inputs(tokens, padded=False) - _, scales = fused_rope_wo_a_bf16(o.view(tokens, 2, -1), weight, freqs, positions) - blocks = torch.arange(N_OUT // 32) + o, weight, freqs, positions = _inputs(tokens, padded=False, groups=groups) + grouped = o.view(tokens, groups, -1) + _, scales = fused_rope_wo_a_bf16(grouped, weight, freqs, positions) + blocks = torch.arange(groups * RANK // 32) rows = torch.arange(tokens, 128)[:, None] assert torch.count_nonzero(scales.cpu()[_sf_index(blocks, rows)]) == 0