[DSV4.1] Enable Hopper decode optimizations for TP8

This commit is contained in:
Yuhao Yang
2026-10-20 10:24:25 +08:00
committed by minke.yu
parent 8c6cbd8c40
commit 4938eea2dc
4 changed files with 107 additions and 5 deletions
@@ -0,0 +1,32 @@
{
"1": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 4,
"SWAP_AB": true,
"SPLIT_K": 1
},
"2": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 4,
"SWAP_AB": true,
"SPLIT_K": 1
},
"64": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 4,
"SWAP_AB": true,
"SPLIT_K": 1
}
}
@@ -0,0 +1,32 @@
{
"1": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 4,
"SWAP_AB": true,
"SPLIT_K": 1
},
"2": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 4,
"SWAP_AB": true,
"SPLIT_K": 1
},
"64": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 4,
"SWAP_AB": true,
"SPLIT_K": 1
}
}
@@ -0,0 +1,32 @@
{
"1": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 4,
"SWAP_AB": true,
"SPLIT_K": 16
},
"2": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 3,
"SWAP_AB": true,
"SPLIT_K": 16
},
"64": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 32,
"num_warps": 4,
"num_stages": 4,
"SWAP_AB": true,
"SPLIT_K": 8
}
}
+11 -5
View File
@@ -467,8 +467,13 @@ def _apply_wo_a_bf16_matmul(
is_prefill: bool = False,
fast_path: bool = False,
) -> torch.Tensor | Mxfp8SwizzledInput:
# o [T, G, D] @ wo_a [G, R, D] -> [T, G, R]; the fast paths below are gated
# on the exact validated TP4 shapes and write token-major output directly.
"""Compute bf16 wo_a: o [T, G, D] @ wo_a [G, R, D] -> [T, G, R].
Single-token decode uses a GEMV for the validated TP4/TP8 shapes. Blackwell
verify batches up to 384 rows write token-major output directly to avoid
the layout copy before wo_b. ROCm decode can use aiter batched GEMM;
other cases use torch.einsum.
"""
global _wo_a_aiter_batched_gemm_disabled
if (
fast_path
@@ -490,12 +495,13 @@ def _apply_wo_a_bf16_matmul(
and get_platform().is_blackwell
)
)
and o.shape[1:] == (2, 4096)
and wo_a.shape == (2, 1024, 4096)
and o.shape[2] == 4096
and o.shape[1] in (1, 2)
and wo_a.shape == (o.shape[1], 1024, 4096)
and o.dtype == wo_a.dtype == torch.bfloat16
and o.stride(2) == 1
and o.stride(1) == 4096
and o.stride(0) >= 8192
and o.stride(0) >= 4096 * o.shape[1]
and wo_a.is_contiguous()
):
if is_decode and o.shape[0] == 1: