[Perf] mhc: Parallelize small-batch post across hidden tiles (#44906)

Co-authored-by: leonardHONG <178642616+leonardHONG@users.noreply.github.com>
Co-authored-by: Xiaoyu Zhang <1182563586@qq.com>
This commit is contained in:
leonardHONG
2026-10-19 15:07:13 +08:00
committed by GitHub
co-authored by leonardHONG Xiaoyu Zhang
parent b6d3b7e163
commit a5117beb11
+20 -4
View File
@@ -1266,7 +1266,16 @@ def mhc_pre(
},
)
def mhc_post_tilelang(
a, b, c, d, x, hc: int, hidden: int, n_thr: int = 128, h_blk: int = 1024
a,
b,
c,
d,
x,
hc: int,
hidden: int,
n_thr: int = 128,
h_blk: int = 1024,
num_h_blocks: int = 1,
):
n = T.dynamic("num_tokens")
h = hidden
@@ -1279,7 +1288,7 @@ def mhc_post_tilelang(
x: T.Tensor((n, hc, h), T.bfloat16)
ENABLE_PDL = is_arch_support_pdl()
with T.Kernel(n, threads=n_thr) as i_n:
with T.Kernel(n, num_h_blocks, threads=n_thr) as (i_n, i_h_block):
if ENABLE_PDL:
T.pdl_sync()
@@ -1296,7 +1305,8 @@ def mhc_post_tilelang(
T.copy(a[i_n, 0, 0], a_local)
T.copy(c[i_n, 0], c_local)
for i0_h in T.Pipelined(T.ceildiv(h, h_blk), num_stages=2):
for i_h_iter in T.Pipelined(T.ceildiv(h, h_blk * num_h_blocks), num_stages=2):
i0_h = i_h_iter * num_h_blocks + i_h_block
T.copy(b[i_n, 0, i0_h * h_blk], b_shared)
T.copy(d[i_n, i0_h * h_blk], d_shared)
@@ -1326,6 +1336,11 @@ def mhc_post(
post_layer_mix = strict_contiguous(post_layer_mix)
comb_res_mix = strict_contiguous(comb_res_mix)
out = torch.empty_like(residual)
hidden_size = residual.shape[-1]
# Small batches need more parallelism than one CTA per token.
num_h_blocks = (
hidden_size // math.gcd(hidden_size, 1024) if x.shape[0] <= 256 else 1
)
mhc_post_tilelang(
comb_res_mix,
residual,
@@ -1333,7 +1348,8 @@ def mhc_post(
x,
out,
residual.shape[-2],
residual.shape[-1],
hidden_size,
num_h_blocks=num_h_blocks,
)
return out