[Perf] DeepSeek-V4.1: Raise the SM100 swap-AB decode cap to 40 rows (#44632)

Co-authored-by: weireweire <20922698+weireweire@users.noreply.github.com>
Co-authored-by: nbommi19 <227267412+nbommi19@users.noreply.github.com>
This commit is contained in:
weireweire
2026-10-19 22:20:10 -07:00
committed by GitHub
co-authored by weireweire nbommi19
parent 553836c164
commit 602c92ccd8
2 changed files with 16 additions and 6 deletions
@@ -11,6 +11,8 @@ from .kv_layout import KVLayout
LAYOUT = KVLayout.V4
MAX_BATCH = 8
# SM100 row cap, below the ~48-row break-even against FlashMLA.
MAX_ROWS = 40
NUM_HEADS = 16
HEAD_DIM = 512
SOFTMAX_SCALE = HEAD_DIM**-0.5
@@ -26,7 +28,7 @@ def can_use_swapab_attention(
) -> bool:
"""The caller adds the SM100 and single-query forward-mode gates."""
return (
0 < q.shape[0] <= MAX_BATCH
0 < q.shape[0] <= MAX_ROWS
and num_heads in (NUM_HEADS, 2 * NUM_HEADS)
and q.dtype == torch.bfloat16
and q.shape[-1] == HEAD_DIM
@@ -119,7 +121,7 @@ def swapab_attention(
block = 64
assert q.ndim in (3, 4) and (q.ndim == 3 or q.shape[1] == 1)
assert 0 < b <= MAX_BATCH and d == HEAD_DIM
assert 0 < b <= MAX_ROWS and d == HEAD_DIM
assert q.dtype == torch.bfloat16 and q.stride(-1) == 1
assert kv.shape[-1] == LAYOUT.bytes_per_token
assert kv.dtype in (torch.uint8, torch.float8_e4m3fn)