[Perf] diffusion: Route dense attention with a sub-tile side to FA4 on Blackwell (#46107)
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Mick Qian
Claude Opus 5.5
parent
de7055f596
commit
42ac81c655
@@ -325,8 +325,12 @@ class DynamicCudnnSDPAImpl(SDPAImpl):
|
||||
if self._is_sm100:
|
||||
# B200/sm_100: cuDNN SDPA measured 1.25-1.5x faster than FA4 CuTe
|
||||
# for dense non-causal diffusion attention, both self-attn
|
||||
# (Sq == Skv, up to S=506K) and cross-attn (Skv = text len).
|
||||
return True
|
||||
# (Sq == Skv, up to S=506K) and cross-attn (Skv = text len). A side
|
||||
# shorter than one 128-row tile while the other is not drops cuDNN to
|
||||
# an sm80 kernel that FA4 beats 2-3x (B300, 126 x 1536 tokens:
|
||||
# 24.9 vs 12.1 us)
|
||||
short_side, long_side = sorted((query.shape[1], key.shape[1]))
|
||||
return short_side >= 128 or long_side < 128
|
||||
if query.shape[1] != key.shape[1]:
|
||||
return False
|
||||
return query.shape[-1] == 64 and query.shape[1] == 1024 and query.shape[0] >= 4
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.runtime.layers.attention.backends.sdpa import (
|
||||
DynamicCudnnSDPAImpl,
|
||||
)
|
||||
|
||||
|
||||
def _qkv(q_len: int, kv_len: int, heads: int = 32, head_dim: int = 64):
|
||||
def tensor(seq_len: int):
|
||||
return SimpleNamespace(
|
||||
shape=(1, seq_len, heads, head_dim),
|
||||
dtype=torch.bfloat16,
|
||||
device=SimpleNamespace(type="cuda"),
|
||||
)
|
||||
|
||||
return tensor(q_len), tensor(kv_len), tensor(kv_len)
|
||||
|
||||
|
||||
class TestDynamicCudnnSDPARouting(unittest.TestCase):
|
||||
def _uses_cudnn(self, q_len: int, kv_len: int) -> bool:
|
||||
impl = SimpleNamespace(causal=False, _cudnn_failed=False, _is_sm100=True)
|
||||
return DynamicCudnnSDPAImpl._use_cudnn_sdpa(impl, *_qkv(q_len, kv_len))
|
||||
|
||||
def test_blackwell_keeps_cudnn_when_both_sides_fill_a_tile(self):
|
||||
for q_len, kv_len in ((128, 128), (1536, 1024), (6144, 6144), (192, 6144)):
|
||||
self.assertTrue(self._uses_cudnn(q_len, kv_len), (q_len, kv_len))
|
||||
|
||||
def test_blackwell_routes_a_side_shorter_than_a_tile_to_fa4(self):
|
||||
for q_len, kv_len in ((126, 1024), (1536, 126), (126, 1536), (32, 4096)):
|
||||
self.assertFalse(self._uses_cudnn(q_len, kv_len), (q_len, kv_len))
|
||||
|
||||
def test_blackwell_keeps_cudnn_when_both_sides_are_short(self):
|
||||
self.assertTrue(self._uses_cudnn(126, 126))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user