diff --git a/python/sglang/multimodal_gen/runtime/layers/attention/backends/sdpa.py b/python/sglang/multimodal_gen/runtime/layers/attention/backends/sdpa.py index 09179ad24a..685297e696 100644 --- a/python/sglang/multimodal_gen/runtime/layers/attention/backends/sdpa.py +++ b/python/sglang/multimodal_gen/runtime/layers/attention/backends/sdpa.py @@ -325,8 +325,12 @@ class DynamicCudnnSDPAImpl(SDPAImpl): if self._is_sm100: # B200/sm_100: cuDNN SDPA measured 1.25-1.5x faster than FA4 CuTe # for dense non-causal diffusion attention, both self-attn - # (Sq == Skv, up to S=506K) and cross-attn (Skv = text len). - return True + # (Sq == Skv, up to S=506K) and cross-attn (Skv = text len). A side + # shorter than one 128-row tile while the other is not drops cuDNN to + # an sm80 kernel that FA4 beats 2-3x (B300, 126 x 1536 tokens: + # 24.9 vs 12.1 us) + short_side, long_side = sorted((query.shape[1], key.shape[1])) + return short_side >= 128 or long_side < 128 if query.shape[1] != key.shape[1]: return False return query.shape[-1] == 64 and query.shape[1] == 1024 and query.shape[0] >= 4 diff --git a/python/sglang/multimodal_gen/test/unit/test_dynamic_cudnn_sdpa_routing.py b/python/sglang/multimodal_gen/test/unit/test_dynamic_cudnn_sdpa_routing.py new file mode 100644 index 0000000000..7c10102d47 --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_dynamic_cudnn_sdpa_routing.py @@ -0,0 +1,40 @@ +import unittest +from types import SimpleNamespace + +import torch + +from sglang.multimodal_gen.runtime.layers.attention.backends.sdpa import ( + DynamicCudnnSDPAImpl, +) + + +def _qkv(q_len: int, kv_len: int, heads: int = 32, head_dim: int = 64): + def tensor(seq_len: int): + return SimpleNamespace( + shape=(1, seq_len, heads, head_dim), + dtype=torch.bfloat16, + device=SimpleNamespace(type="cuda"), + ) + + return tensor(q_len), tensor(kv_len), tensor(kv_len) + + +class TestDynamicCudnnSDPARouting(unittest.TestCase): + def _uses_cudnn(self, q_len: int, kv_len: int) -> bool: + impl = SimpleNamespace(causal=False, _cudnn_failed=False, _is_sm100=True) + return DynamicCudnnSDPAImpl._use_cudnn_sdpa(impl, *_qkv(q_len, kv_len)) + + def test_blackwell_keeps_cudnn_when_both_sides_fill_a_tile(self): + for q_len, kv_len in ((128, 128), (1536, 1024), (6144, 6144), (192, 6144)): + self.assertTrue(self._uses_cudnn(q_len, kv_len), (q_len, kv_len)) + + def test_blackwell_routes_a_side_shorter_than_a_tile_to_fa4(self): + for q_len, kv_len in ((126, 1024), (1536, 126), (126, 1536), (32, 4096)): + self.assertFalse(self._uses_cudnn(q_len, kv_len), (q_len, kv_len)) + + def test_blackwell_keeps_cudnn_when_both_sides_are_short(self): + self.assertTrue(self._uses_cudnn(126, 126)) + + +if __name__ == "__main__": + unittest.main()