[Perf] diffusion: Route dense attention with a sub-tile side to FA4 on Blackwell (#46107)

Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Mick
2026-10-24 10:59:25 +08:00
committed by GitHub
co-authored by Mick Qian Claude Opus 5.5
parent de7055f596
commit 42ac81c655
2 changed files with 46 additions and 2 deletions
@@ -325,8 +325,12 @@ class DynamicCudnnSDPAImpl(SDPAImpl):
if self._is_sm100:
# B200/sm_100: cuDNN SDPA measured 1.25-1.5x faster than FA4 CuTe
# for dense non-causal diffusion attention, both self-attn
# (Sq == Skv, up to S=506K) and cross-attn (Skv = text len).
return True
# (Sq == Skv, up to S=506K) and cross-attn (Skv = text len). A side
# shorter than one 128-row tile while the other is not drops cuDNN to
# an sm80 kernel that FA4 beats 2-3x (B300, 126 x 1536 tokens:
# 24.9 vs 12.1 us)
short_side, long_side = sorted((query.shape[1], key.shape[1]))
return short_side >= 128 or long_side < 128
if query.shape[1] != key.shape[1]:
return False
return query.shape[-1] == 64 and query.shape[1] == 1024 and query.shape[0] >= 4
@@ -0,0 +1,40 @@
import unittest
from types import SimpleNamespace
import torch
from sglang.multimodal_gen.runtime.layers.attention.backends.sdpa import (
DynamicCudnnSDPAImpl,
)
def _qkv(q_len: int, kv_len: int, heads: int = 32, head_dim: int = 64):
def tensor(seq_len: int):
return SimpleNamespace(
shape=(1, seq_len, heads, head_dim),
dtype=torch.bfloat16,
device=SimpleNamespace(type="cuda"),
)
return tensor(q_len), tensor(kv_len), tensor(kv_len)
class TestDynamicCudnnSDPARouting(unittest.TestCase):
def _uses_cudnn(self, q_len: int, kv_len: int) -> bool:
impl = SimpleNamespace(causal=False, _cudnn_failed=False, _is_sm100=True)
return DynamicCudnnSDPAImpl._use_cudnn_sdpa(impl, *_qkv(q_len, kv_len))
def test_blackwell_keeps_cudnn_when_both_sides_fill_a_tile(self):
for q_len, kv_len in ((128, 128), (1536, 1024), (6144, 6144), (192, 6144)):
self.assertTrue(self._uses_cudnn(q_len, kv_len), (q_len, kv_len))
def test_blackwell_routes_a_side_shorter_than_a_tile_to_fa4(self):
for q_len, kv_len in ((126, 1024), (1536, 126), (126, 1536), (32, 4096)):
self.assertFalse(self._uses_cudnn(q_len, kv_len), (q_len, kv_len))
def test_blackwell_keeps_cudnn_when_both_sides_are_short(self):
self.assertTrue(self._uses_cudnn(126, 126))
if __name__ == "__main__":
unittest.main()