[Perf] DeepSeek-V4.1: Run the small-batch decode fusions at TP2 (#43838)
Co-authored-by: weireweire <20922698+weireweire@users.noreply.github.com> Co-authored-by: nbommi19 <227267412+nbommi19@users.noreply.github.com> Co-authored-by: Po-Han Huang <pohanh@nvidia.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Po-Han Huang (NVIDIA) <53919306+nvpohanh@users.noreply.github.com>
This commit is contained in:
co-authored by
weireweire
nbommi19
Po-Han Huang
Claude Opus 5.5
Po-Han Huang
parent
fdd53418a0
commit
72d886efb8
@@ -3,12 +3,12 @@
|
||||
// Collapses three launches on the DSV4 decode/verify path (fused_rope_inplace,
|
||||
// _wo_a_partial, _wo_a_reduce_quant) into one cluster-launched kernel.
|
||||
//
|
||||
// x [T, 2, 4096] bf16 (a strided view of a [T, 64, 512] attention output)
|
||||
// W [2, 1024, 4096] bf16
|
||||
// -> q [T, 2048] e4m3 + scales [8192] ue8m0, and/or y [T, 2048] bf16
|
||||
// x [T, G, 4096] bf16 (a strided view of a [T, 64, 512] attention output)
|
||||
// W [G, 1024, 4096] bf16
|
||||
// -> q [T, G*1024] e4m3 + scales [G*4096] ue8m0, and/or y [T, G*1024] bf16
|
||||
//
|
||||
// Decomposition: the 2048 output columns split into 32 N-tiles of 64, one per
|
||||
// cluster; K splits 8 ways across the cluster's CTAs, so each CTA owns exactly
|
||||
// Decomposition: the G*1024 output columns split into G*16 N-tiles of 64, one
|
||||
// per cluster; K splits 8 ways across the cluster's CTAs, so each CTA owns exactly
|
||||
// one of the group's 8 heads -- and therefore exactly one 64-wide RoPE window,
|
||||
// which is exactly one staged K-tile. That is what folds the rope launch in
|
||||
// without a grid-wide dependency.
|
||||
@@ -251,17 +251,19 @@ namespace sglang::wo_a_fused {
|
||||
// Shape
|
||||
// ---------------------------------------------------------------------------
|
||||
/**
|
||||
* \brief Compile-time shape for the TP4 WO-A decomposition.
|
||||
* \brief Compile-time shape for the WO-A decomposition.
|
||||
*
|
||||
* \tparam kMaxTokens Token capacity; also the MMA's N axis, so a multiple of 8.
|
||||
* \tparam kGroups Local o_groups: 2 at TP4, 4 at TP2.
|
||||
*
|
||||
* 2048 output columns split into kClusters N-tiles of kNTile, one per cluster;
|
||||
* K splits kSplitK ways across the cluster's CTAs, giving each CTA exactly one
|
||||
* of the group's 8 heads and therefore exactly one RoPE window.
|
||||
* kGroups x 1024 output columns split into kClusters N-tiles of kNTile, one per
|
||||
* cluster; K splits kSplitK ways across the cluster's CTAs, giving each CTA
|
||||
* exactly one of the group's 8 heads and therefore exactly one RoPE window.
|
||||
*/
|
||||
template <int kMaxTokens>
|
||||
template <int kMaxTokens, int kGroups = 2>
|
||||
struct WoATrait {
|
||||
static_assert(kMaxTokens == 16 || kMaxTokens == 32, "only the x16/x32 TMEM loads are wired up");
|
||||
static_assert(kGroups == 2 || kGroups == 4, "TP4 (2 local groups) and TP2 (4) only");
|
||||
|
||||
static constexpr int kMMax = kMaxTokens;
|
||||
static constexpr int kNTile = 64; // == 2 MXFP8 blocks
|
||||
@@ -274,7 +276,8 @@ struct WoATrait {
|
||||
static constexpr int kHeadStages = kHeadDim / kBlockK;
|
||||
static constexpr int kHeadsPerCta = kKSlab / kHeadDim;
|
||||
static constexpr int kRopeDim = 64;
|
||||
static constexpr int kNOut = 2048; // 2 groups x 1024
|
||||
static constexpr int kGroupN = 1024; // o_lora_rank
|
||||
static constexpr int kNOut = kGroups * kGroupN;
|
||||
// One UE8M0 byte per MXFP8 block, for 128 padded token rows.
|
||||
static constexpr int kScaleBytes = (kNOut / 32) * 128;
|
||||
static constexpr int kClusters = kNOut / kNTile;
|
||||
@@ -315,7 +318,8 @@ struct WoATrait {
|
||||
uint64_t tmem_ld_done;
|
||||
uint32_t taddr;
|
||||
};
|
||||
// 2 CTAs/SM needs <= 113 KiB each; the 32-cluster wave depends on it.
|
||||
// 2 CTAs/SM needs <= 113 KiB each; the single-wave TP4 grid (32 clusters) depends on it.
|
||||
// TP2 launches 64 clusters / 512 CTAs, ~1.7 waves at 2 CTAs/SM, which likely bounds its WO-A gain.
|
||||
static_assert(sizeof(Smem) <= 115712, "smem would drop the kernel to 1 CTA/SM");
|
||||
};
|
||||
|
||||
@@ -360,9 +364,9 @@ __global__ void __cluster_dims__(Trait::kSplitK, 1, 1) __launch_bounds__(Trait::
|
||||
const __grid_constant__ CUtensorMap x_map,
|
||||
const float* __restrict__ freqs, // [S, 64] fp32, (cos0, sin0, cos1, ...)
|
||||
const PosT* __restrict__ positions, // [T] int32 or int64
|
||||
fp8_e4m3_t* __restrict__ q, // [T, 2048], may be null
|
||||
uint8_t* __restrict__ scales_out, // [8192], may be null when q is
|
||||
bf16_t* __restrict__ y, // [T, 2048], may be null
|
||||
fp8_e4m3_t* __restrict__ q, // [T, kNOut], may be null
|
||||
uint8_t* __restrict__ scales_out, // [kScaleBytes], may be null when q is
|
||||
bf16_t* __restrict__ y, // [T, kNOut], may be null
|
||||
uint32_t m) {
|
||||
using namespace device;
|
||||
// The trait's shape, unqualified, so the body below reads as plain constants.
|
||||
@@ -377,6 +381,7 @@ __global__ void __cluster_dims__(Trait::kSplitK, 1, 1) __launch_bounds__(Trait::
|
||||
constexpr int kHeadsPerCta = Trait::kHeadsPerCta;
|
||||
constexpr int kRopeDim = Trait::kRopeDim;
|
||||
constexpr int kNOut = Trait::kNOut;
|
||||
constexpr int kGroupN = Trait::kGroupN;
|
||||
constexpr int kWarps = Trait::kWarps;
|
||||
constexpr int kThreads = Trait::kThreads;
|
||||
constexpr int kDrainWarps = Trait::kDrainWarps;
|
||||
@@ -401,8 +406,8 @@ __global__ void __cluster_dims__(Trait::kSplitK, 1, 1) __launch_bounds__(Trait::
|
||||
const uint32_t lane_id = tx % kWarpThreads;
|
||||
const uint32_t rank = ptx::cluster_ctarank(); // == the head this CTA owns
|
||||
const uint32_t work_id = blockIdx.x / kSplitK;
|
||||
const uint32_t group = work_id / (1024 / kNTile);
|
||||
const uint32_t n0 = (work_id % (1024 / kNTile)) * kNTile;
|
||||
const uint32_t group = work_id / (kGroupN / kNTile);
|
||||
const uint32_t n0 = (work_id % (kGroupN / kNTile)) * kNTile;
|
||||
const uint32_t k0 = rank * kKSlab;
|
||||
|
||||
// Before anything else: the descriptor fetch is serial with the first tile's
|
||||
@@ -442,12 +447,12 @@ __global__ void __cluster_dims__(Trait::kSplitK, 1, 1) __launch_bounds__(Trait::
|
||||
|
||||
if (warp_id == TMA_W) {
|
||||
// The weights are not the producer kernel's output, so this warp never waits
|
||||
// on PDL and the whole 16 MiB stream overlaps the attention tail.
|
||||
// on PDL and the whole weight stream overlaps the attention tail.
|
||||
if (warp::elect_one_lane()) {
|
||||
#pragma unroll
|
||||
for (int s = 0; s < kStages; ++s) {
|
||||
ptx::cp_async_bulk_tensor_2d(
|
||||
ptx::to_shared(smem.w[s]), &w_map, k0 + s * kBlockK, group * 1024 + n0, &smem.w_ready[s]);
|
||||
ptx::to_shared(smem.w[s]), &w_map, k0 + s * kBlockK, group * kGroupN + n0, &smem.w_ready[s]);
|
||||
ptx::mbar_arrive_expect_tx(&smem.w_ready[s], kWTile);
|
||||
}
|
||||
}
|
||||
@@ -582,7 +587,7 @@ __global__ void __cluster_dims__(Trait::kSplitK, 1, 1) __launch_bounds__(Trait::
|
||||
|
||||
// Only the first kEpiWarps warps own a token; inbox's middle dim is
|
||||
// kTokPerRank, and t would otherwise collide with the next rank's tokens.
|
||||
const auto col0 = group * 1024 + n0;
|
||||
const auto col0 = group * kGroupN + n0;
|
||||
const auto t = rank * kTokPerRank + warp_id;
|
||||
// Unconditional, and before the guard: this is what keeps the CTA alive
|
||||
// until every peer's st.async into our inbox has landed. A rank whose tokens
|
||||
@@ -690,19 +695,20 @@ namespace sglang {
|
||||
/**
|
||||
* \brief Fused inverse-RoPE + grouped WO-A BF16 GEMM + MXFP8 quantization.
|
||||
*
|
||||
* \param x [T, 2, 4096] bf16 attention output; a strided view of a
|
||||
* \tparam kGroups Local o_groups G.
|
||||
* \param x [T, G, 4096] bf16 attention output; a strided view of a
|
||||
* [T, 64, 512] buffer, so only the group and K axes need to be
|
||||
* contiguous and the row stride is read off the tensor.
|
||||
* \param w [2, 1024, 4096] bf16, contiguous.
|
||||
* \param w [G, 1024, 4096] bf16, contiguous.
|
||||
* \param freqs [S, 64] fp32 rotation table, (cos0, sin0, cos1, sin1, ...).
|
||||
* \param positions [T] absolute token positions, int32 or int64.
|
||||
* Exactly one output form must be supplied: either (\p q, \p scales) or \p y.
|
||||
*
|
||||
* \param q [T, 2048] e4m3 quantized output; pairs with \p scales.
|
||||
* \param scales [8192] uint8 UE8M0 scales in FlashInfer's 128x4 swizzle.
|
||||
* \param y [T, 2048] bf16, the result before quantization.
|
||||
* \param q [T, G*1024] e4m3 quantized output; pairs with \p scales.
|
||||
* \param scales [G*4096] uint8 UE8M0 scales in FlashInfer's 128x4 swizzle.
|
||||
* \param y [T, G*1024] bf16, the result before quantization.
|
||||
*/
|
||||
template <int kMaxTokens>
|
||||
template <int kMaxTokens, int kGroups = 2>
|
||||
inline void wo_a_fused_run(
|
||||
tvm::ffi::TensorView x,
|
||||
tvm::ffi::TensorView w,
|
||||
@@ -713,7 +719,7 @@ inline void wo_a_fused_run(
|
||||
tvm::ffi::Optional<tvm::ffi::TensorView> y) {
|
||||
using namespace host;
|
||||
using namespace wo_a_fused; // make_map, wo_a_fused_kernel
|
||||
using Trait = WoATrait<kMaxTokens>;
|
||||
using Trait = WoATrait<kMaxTokens, kGroups>;
|
||||
constexpr int kMMax = Trait::kMMax;
|
||||
constexpr int kNTile = Trait::kNTile;
|
||||
constexpr int kBlockK = Trait::kBlockK;
|
||||
@@ -728,17 +734,17 @@ inline void wo_a_fused_run(
|
||||
|
||||
SymbolicSize T = {"num_tokens"};
|
||||
// The row stride is captured rather than required: it is 32768 elements when
|
||||
// the attention backend pads to 64 heads and 8192 when it does not.
|
||||
// the attention backend pads to 64 heads and 512 x local heads when it does not.
|
||||
SymbolicSize XS0 = {"x_row_stride"};
|
||||
SymbolicDevice device_;
|
||||
device_.set_options<kDLCUDA>();
|
||||
|
||||
TensorMatcher({T, 2, kGroupK}) //
|
||||
TensorMatcher({T, kGroups, kGroupK}) //
|
||||
.with_strides({XS0, kGroupK, 1})
|
||||
.with_dtype<bf16_t>()
|
||||
.with_device<kDLCUDA>(device_)
|
||||
.verify(x);
|
||||
TensorMatcher({2, kNOut / 2, kGroupK}) //
|
||||
TensorMatcher({kGroups, kNOut / kGroups, kGroupK}) //
|
||||
.with_dtype<bf16_t>()
|
||||
.with_device<kDLCUDA>(device_)
|
||||
.verify(w);
|
||||
@@ -774,9 +780,10 @@ inline void wo_a_fused_run(
|
||||
const int64_t x_stride0 = XS0.unwrap();
|
||||
const DLDevice device = device_.unwrap();
|
||||
CHECK_HOST(m >= 1 && m <= kMMax) << "wo_a_fused supports 1 <= T <= " << kMMax << ", got " << m;
|
||||
CHECK_HOST(x_stride0 >= 2 * kGroupK) << "x row stride must cover both groups, got " << x_stride0;
|
||||
CHECK_HOST(x_stride0 >= kGroups * kGroupK)
|
||||
<< "x row stride must cover all " << kGroups << " groups, got " << x_stride0;
|
||||
const auto w_map = make_map(w.data_ptr(), kGroupK, kNOut, kGroupK * 2, kBlockK, kNTile);
|
||||
const auto x_map = make_map(x.data_ptr(), 2 * kGroupK, m, x_stride0 * 2, kBlockK, kMMax);
|
||||
const auto x_map = make_map(x.data_ptr(), kGroups * kGroupK, m, x_stride0 * 2, kBlockK, kMMax);
|
||||
const auto launch = [&](const auto* pos) {
|
||||
using PosT = std::decay_t<decltype(*pos)>;
|
||||
constexpr auto kernel = wo_a_fused_kernel<Trait, PosT>;
|
||||
|
||||
@@ -27,7 +27,7 @@ def can_use_swapab_attention(
|
||||
"""The caller adds the SM100 and single-query forward-mode gates."""
|
||||
return (
|
||||
0 < q.shape[0] <= MAX_BATCH
|
||||
and num_heads == NUM_HEADS
|
||||
and num_heads in (NUM_HEADS, 2 * NUM_HEADS)
|
||||
and q.dtype == torch.bfloat16
|
||||
and q.shape[-1] == HEAD_DIM
|
||||
and head_dim_v == HEAD_DIM
|
||||
@@ -106,16 +106,20 @@ def swapab_attention(
|
||||
inv_rope=None,
|
||||
partial_gluon=None,
|
||||
):
|
||||
"""V4-layout attention on 16 heads; `extra_*` is a second slot range appended
|
||||
to each request's keys, and the attention sink is folded in exactly once.
|
||||
"""V4-layout attention on 16 heads, or 32 as two head groups (SM100 kernel
|
||||
only); `extra_*` is a second slot range appended to each request's keys, and
|
||||
the attention sink is folded in exactly once.
|
||||
`partial_gluon` replaces the SM100 split-KV kernel (swapab_gluon_hip passes gfx950's)."""
|
||||
b, h, d = q.shape[0], q.shape[-2], q.shape[-1]
|
||||
head_groups = h // NUM_HEADS
|
||||
assert h == NUM_HEADS or (h == 2 * NUM_HEADS and partial_gluon is None)
|
||||
group_kwargs = {"HG": head_groups} if head_groups > 1 else {}
|
||||
if partial_gluon is None:
|
||||
from .decode_attention_sm100_gluon import partial_gluon
|
||||
|
||||
block = 64
|
||||
b, h, d = q.shape[0], q.shape[-2], q.shape[-1]
|
||||
assert q.ndim in (3, 4) and (q.ndim == 3 or q.shape[1] == 1)
|
||||
assert 0 < b <= MAX_BATCH and h == NUM_HEADS and d == HEAD_DIM
|
||||
assert 0 < b <= MAX_BATCH and d == HEAD_DIM
|
||||
assert q.dtype == torch.bfloat16 and q.stride(-1) == 1
|
||||
assert kv.shape[-1] == LAYOUT.bytes_per_token
|
||||
assert kv.dtype in (torch.uint8, torch.float8_e4m3fn)
|
||||
@@ -138,7 +142,7 @@ def swapab_attention(
|
||||
maximum = torch.empty((b, nt, h), dtype=torch.float32, device=q.device)
|
||||
sums = torch.empty_like(maximum)
|
||||
out = torch.empty((b, h, 512), dtype=q.dtype, device=q.device)
|
||||
partial_gluon[(b, nt)](
|
||||
partial_gluon[(b, nt, head_groups)](
|
||||
q,
|
||||
kv,
|
||||
extra_kv,
|
||||
@@ -162,7 +166,7 @@ def swapab_attention(
|
||||
NT=nt,
|
||||
KT=kt,
|
||||
BT=block,
|
||||
H=h,
|
||||
H=NUM_HEADS,
|
||||
SCALE=SOFTMAX_SCALE,
|
||||
KTOKENS=kv.shape[0] * kv.shape[1],
|
||||
ETOKENS=extra_kv.shape[0] * extra_kv.shape[1],
|
||||
@@ -172,6 +176,7 @@ def swapab_attention(
|
||||
SCALE_BYTES=LAYOUT.scale_bytes,
|
||||
TILE=LAYOUT.tile_size,
|
||||
num_warps=4,
|
||||
**group_kwargs,
|
||||
)
|
||||
if inv_rope is not None:
|
||||
freqs, positions = inv_rope
|
||||
|
||||
@@ -98,9 +98,12 @@ def partial_gluon(
|
||||
DATA_BYTES: gl.constexpr,
|
||||
SCALE_BYTES: gl.constexpr,
|
||||
TILE: gl.constexpr,
|
||||
HG: gl.constexpr = 1,
|
||||
):
|
||||
gl.static_assert(SWAP_AB and H == 16 and (BT == 64 or BT == 128))
|
||||
gl.static_assert(SWAP_AB and H == 16 and (BT == 64 or BT == 128) and HG >= 1)
|
||||
HT: gl.constexpr = H * HG
|
||||
b, t = gl.program_id(0), gl.program_id(1)
|
||||
h0 = gl.program_id(2) * H
|
||||
kv_layout: gl.constexpr = gl.BlockedLayout([1, 8], [4, 8], [4, 1], [1, 0])
|
||||
n = gl.arange(0, BT, gl.SliceLayout(1, kv_layout))
|
||||
if t < KT:
|
||||
@@ -137,7 +140,7 @@ def partial_gluon(
|
||||
)
|
||||
qh = gl.arange(0, H, gl.SliceLayout(1, kv_layout))
|
||||
qd = gl.arange(0, 512, gl.SliceLayout(0, kv_layout))
|
||||
q = gl.load(Q + b * QS + qh[:, None] * QH + qd[None, :])
|
||||
q = gl.load(Q + b * QS + (h0 + qh[:, None]) * QH + qd[None, :])
|
||||
q_smem = gl.allocate_shared_memory(
|
||||
gl.bfloat16,
|
||||
[H, 512],
|
||||
@@ -199,8 +202,8 @@ def partial_gluon(
|
||||
value_hd = gl.convert_layout(value.permute((1, 0)), hd_layout)
|
||||
h = gl.arange(0, H, gl.SliceLayout(1, hd_layout))
|
||||
d = gl.arange(0, 512, gl.SliceLayout(0, hd_layout))
|
||||
gl.store(PART + ((b * NT + t) * H + h[:, None]) * 512 + d[None, :], value_hd)
|
||||
gl.store(PART + ((b * NT + t) * HT + h0 + h[:, None]) * 512 + d[None, :], value_hd)
|
||||
stat_layout: gl.constexpr = gl.BlockedLayout([1], [32], [4], [0])
|
||||
hs = gl.arange(0, H, stat_layout)
|
||||
gl.store(MAX + (b * NT + t) * H + hs, gl.convert_layout(mx, stat_layout))
|
||||
gl.store(SUM + (b * NT + t) * H + hs, gl.convert_layout(denom, stat_layout))
|
||||
gl.store(MAX + (b * NT + t) * HT + h0 + hs, gl.convert_layout(mx, stat_layout))
|
||||
gl.store(SUM + (b * NT + t) * HT + h0 + hs, gl.convert_layout(denom, stat_layout))
|
||||
|
||||
@@ -9,19 +9,21 @@ from sglang.kernels.jit.utils import cache_once, cuda_stubs_dir, load_jit
|
||||
if TYPE_CHECKING:
|
||||
from tvm_ffi.module import Module
|
||||
|
||||
GROUPS = 2
|
||||
GROUPS = (2, 4)
|
||||
RANK = 1024
|
||||
N_OUT = GROUPS * RANK
|
||||
SCALE_BYTES = 8192
|
||||
MAX_M = 32
|
||||
|
||||
|
||||
def can_use_fused_wo_a(n_local_groups: int, o_lora_rank: int) -> bool:
|
||||
return n_local_groups in GROUPS and o_lora_rank == RANK
|
||||
|
||||
|
||||
@cache_once
|
||||
def _jit_module(max_tokens: int) -> Module:
|
||||
def _jit_module(max_tokens: int, groups: int) -> Module:
|
||||
return load_jit(
|
||||
f"fused_rope_wo_a_m{max_tokens}",
|
||||
f"fused_rope_wo_a_g{groups}_m{max_tokens}",
|
||||
cuda_files=["deepseek_v4/wo_a_fused.cuh"],
|
||||
cuda_wrappers=[("run", f"wo_a_fused_run<{max_tokens}>")],
|
||||
cuda_wrappers=[("run", f"wo_a_fused_run<{max_tokens}, {groups}>")],
|
||||
extra_cuda_cflags=["-O3"],
|
||||
extra_ldflags=[f"-L{cuda_stubs_dir()}", "-lcuda"],
|
||||
)
|
||||
@@ -37,15 +39,19 @@ def fused_rope_wo_a_bf16(
|
||||
) -> Sequence[torch.Tensor]:
|
||||
"""Inverse RoPE + BF16 WO-A, returning BF16 or FlashInfer-swizzled MXFP8."""
|
||||
num_tokens = x.shape[0]
|
||||
groups = weight.shape[0]
|
||||
assert num_tokens <= MAX_M
|
||||
assert groups in GROUPS
|
||||
n_out = groups * RANK
|
||||
max_tokens = 16 if num_tokens <= 16 else 32
|
||||
module = _jit_module(max_tokens)
|
||||
module = _jit_module(max_tokens, groups)
|
||||
if out_mxfp8:
|
||||
q = torch.empty((num_tokens, N_OUT), dtype=torch.float8_e4m3fn, device=x.device)
|
||||
scales = torch.empty(SCALE_BYTES, dtype=torch.uint8, device=x.device)
|
||||
q = torch.empty((num_tokens, n_out), dtype=torch.float8_e4m3fn, device=x.device)
|
||||
# One UE8M0 byte per 32-column block, for 128 padded token rows.
|
||||
scales = torch.empty(n_out // 32 * 128, dtype=torch.uint8, device=x.device)
|
||||
module.run(x, weight, freqs_cis, positions, q, scales, None)
|
||||
return q, scales
|
||||
else:
|
||||
y = x.new_empty((num_tokens, N_OUT))
|
||||
y = x.new_empty((num_tokens, n_out))
|
||||
module.run(x, weight, freqs_cis, positions, None, None, y)
|
||||
return (y,)
|
||||
|
||||
@@ -4119,7 +4119,7 @@ class DeepseekV4AttnBackend(
|
||||
)
|
||||
|
||||
return swapab_attention(
|
||||
q[..., :16, :],
|
||||
q[..., : layer.tp_q_head_num, :],
|
||||
swa_k_cache,
|
||||
swa_page_indices,
|
||||
swa_topk_lengths,
|
||||
|
||||
@@ -31,7 +31,10 @@ from sglang.kernels.ops.attention.dsv4 import (
|
||||
fused_rope_inplace,
|
||||
)
|
||||
from sglang.kernels.ops.attention.dsv4.wo_a import MAX_M as _FUSED_WO_A_MAX_TOKENS
|
||||
from sglang.kernels.ops.attention.dsv4.wo_a import fused_rope_wo_a_bf16
|
||||
from sglang.kernels.ops.attention.dsv4.wo_a import (
|
||||
can_use_fused_wo_a,
|
||||
fused_rope_wo_a_bf16,
|
||||
)
|
||||
from sglang.kernels.ops.attention.flash_mla_sm120 import SM120_DECODE_MAX_TOKENS
|
||||
from sglang.kernels.ops.gemm.dsv4_wo_a import (
|
||||
wo_a_bf16_gemv,
|
||||
@@ -1365,7 +1368,7 @@ class MQALayer(MqaAttentionBase):
|
||||
and not self.use_npu_arch35_mxfp8_wo_a
|
||||
and self.wo_a.weight.dtype == torch.bfloat16
|
||||
and self.wo_a.weight.shape == (self.n_local_groups * self.o_lora_rank, 4096)
|
||||
and (self.n_local_groups, self.o_lora_rank) == (2, 1024)
|
||||
and can_use_fused_wo_a(self.n_local_groups, self.o_lora_rank)
|
||||
)
|
||||
if _is_hip:
|
||||
_hip.init_mqa_layer(self, quant_config)
|
||||
@@ -3324,7 +3327,7 @@ class DeepseekV4DecoderLayerBase(nn.Module):
|
||||
def _can_fuse_ffn_mhc(self) -> bool:
|
||||
"""The MoE counterpart of `_can_fuse_attn_mhc`."""
|
||||
return (
|
||||
self.mlp.tp_size == 4
|
||||
mhc.fused_mhc_tp_ok(self.mlp.tp_size)
|
||||
and get_moe_a2a_backend().is_none()
|
||||
and not self.mlp._shared_expert_tp1
|
||||
and not should_skip_post_experts_all_reduce(is_tp_path=True)
|
||||
@@ -3446,7 +3449,8 @@ class DeepseekV4CudaDecoderLayer(DeepseekV4DecoderLayerBase):
|
||||
self.post_attention_layernorm, accepts_mxfp8=False, hc=self.ffn_hc
|
||||
)
|
||||
return (
|
||||
get_parallel().tp_size == self.self_attn.attn_tp_size == 4
|
||||
get_parallel().tp_size == self.self_attn.attn_tp_size
|
||||
and mhc.fused_mhc_tp_ok(self.self_attn.attn_tp_size)
|
||||
and boundary.norm_fusable
|
||||
and mhc.can_fuse_post(self.hc_cfg)
|
||||
)
|
||||
@@ -4870,7 +4874,7 @@ class DeepseekV4ForCausalLM(nn.Module):
|
||||
|
||||
# The only deployments whose fused-collective gates can fire; registration
|
||||
# must happen eagerly here, before graph capture.
|
||||
if get_platform().is_blackwell and get_parallel().tp_size == 4:
|
||||
if get_platform().is_blackwell and mhc.fused_mhc_tp_ok(get_parallel().tp_size):
|
||||
from sglang.srt.layers.quantization.mxfp4_flashinfer_trtllm_moe import (
|
||||
register_fused_all_reduce_comm,
|
||||
)
|
||||
|
||||
@@ -45,6 +45,14 @@ _is_hip = is_hip()
|
||||
_is_npu = is_npu()
|
||||
_is_xpu = is_xpu()
|
||||
|
||||
# The fused all-reduce epilogues have world-size-templated kernels for TP2 and TP4.
|
||||
_FUSED_MHC_TP_SIZES = (2, 4)
|
||||
|
||||
|
||||
def fused_mhc_tp_ok(tp_size: int) -> bool:
|
||||
return tp_size in _FUSED_MHC_TP_SIZES
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# What a layer hands this module
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
Reference in New Issue
Block a user