[Perf] diffusion: Serve FA4 head sizes on Blackwell instead of falling back to sm80 SDPA (#46102)
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Mick Qian
Claude Opus 5.5
parent
386fa9ecfd
commit
2e6d87e2ca
@@ -779,7 +779,11 @@ class CudaPlatformBase(Platform):
|
||||
)
|
||||
|
||||
supported_sizes = FlashAttentionBackend.get_supported_head_sizes()
|
||||
if head_size not in supported_sizes:
|
||||
# FA4 on sm100 serves every multiple of 8 up to 128, e.g. Qwen-VL vision heads (72, 80)
|
||||
fa4_head_size = (
|
||||
cls.is_blackwell() and head_size <= 128 and head_size % 8 == 0
|
||||
)
|
||||
if head_size not in supported_sizes and not fa4_head_size:
|
||||
logger.info(
|
||||
"Cannot use FlashAttention backend for head size %d.",
|
||||
head_size,
|
||||
|
||||
@@ -200,6 +200,49 @@ class TestCudaAttentionBackendSelection(unittest.TestCase):
|
||||
"sglang.multimodal_gen.runtime.layers.attention.backends.sdpa.DynamicCudnnSDPABackend",
|
||||
)
|
||||
|
||||
def test_blackwell_serves_fa4_head_sizes_outside_the_fa3_list(self):
|
||||
FakeCudaPlatform.is_blackwell_device = True
|
||||
fa_cls_str = "sglang.multimodal_gen.runtime.layers.attention.backends.flash_attn.FlashAttentionBackend"
|
||||
dynamic_cls_str = "sglang.multimodal_gen.runtime.layers.attention.backends.sdpa.DynamicCudnnSDPABackend"
|
||||
|
||||
with patch.object(
|
||||
FakeCudaPlatform,
|
||||
"_prepare_flash_attention_for_blackwell",
|
||||
return_value=True,
|
||||
):
|
||||
for head_size in (72, 80):
|
||||
with self.subTest(head_size=head_size):
|
||||
self.assertEqual(
|
||||
FakeCudaPlatform.get_attn_backend_cls_str(
|
||||
None, head_size, torch.bfloat16
|
||||
),
|
||||
dynamic_cls_str,
|
||||
)
|
||||
self.assertEqual(
|
||||
FakeCudaPlatform.get_attn_backend_cls_str(
|
||||
AttentionBackendEnum.FA, head_size, torch.bfloat16
|
||||
),
|
||||
fa_cls_str,
|
||||
)
|
||||
for head_size in (68, 136):
|
||||
with self.subTest(head_size=head_size):
|
||||
self.assertEqual(
|
||||
FakeCudaPlatform.get_attn_backend_cls_str(
|
||||
None, head_size, torch.bfloat16
|
||||
),
|
||||
SDPA_BACKEND_CLS_STR,
|
||||
)
|
||||
|
||||
def test_pre_blackwell_keeps_torch_sdpa_for_head_size_72(self):
|
||||
for selected_backend in (None, AttentionBackendEnum.FA):
|
||||
with self.subTest(selected_backend=selected_backend):
|
||||
self.assertEqual(
|
||||
FakeCudaPlatform.get_attn_backend_cls_str(
|
||||
selected_backend, 72, torch.bfloat16
|
||||
),
|
||||
SDPA_BACKEND_CLS_STR,
|
||||
)
|
||||
|
||||
def test_invalid_backend_raises(self):
|
||||
with self.assertRaisesRegex(ValueError, "Invalid attention backend"):
|
||||
self.resolve(AttentionBackendEnum.AITER_SAGE)
|
||||
|
||||
Reference in New Issue
Block a user