From 2e6d87e2ca8cdd292fbbe6516c0a341bcf687fd4 Mon Sep 17 00:00:00 2001 From: Mick Date: Sat, 24 Oct 2026 09:20:32 +0800 Subject: [PATCH] [Perf] diffusion: Serve FA4 head sizes on Blackwell instead of falling back to sm80 SDPA (#46102) Co-authored-by: Mick Qian Co-authored-by: Claude Opus 5.5 --- .../multimodal_gen/runtime/platforms/cuda.py | 6 ++- .../test/unit/test_cuda_attention_backend.py | 43 +++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/python/sglang/multimodal_gen/runtime/platforms/cuda.py b/python/sglang/multimodal_gen/runtime/platforms/cuda.py index ead2029c1f..67062fae40 100644 --- a/python/sglang/multimodal_gen/runtime/platforms/cuda.py +++ b/python/sglang/multimodal_gen/runtime/platforms/cuda.py @@ -779,7 +779,11 @@ class CudaPlatformBase(Platform): ) supported_sizes = FlashAttentionBackend.get_supported_head_sizes() - if head_size not in supported_sizes: + # FA4 on sm100 serves every multiple of 8 up to 128, e.g. Qwen-VL vision heads (72, 80) + fa4_head_size = ( + cls.is_blackwell() and head_size <= 128 and head_size % 8 == 0 + ) + if head_size not in supported_sizes and not fa4_head_size: logger.info( "Cannot use FlashAttention backend for head size %d.", head_size, diff --git a/python/sglang/multimodal_gen/test/unit/test_cuda_attention_backend.py b/python/sglang/multimodal_gen/test/unit/test_cuda_attention_backend.py index 9ec7fd8dc5..214fd1d3a7 100644 --- a/python/sglang/multimodal_gen/test/unit/test_cuda_attention_backend.py +++ b/python/sglang/multimodal_gen/test/unit/test_cuda_attention_backend.py @@ -200,6 +200,49 @@ class TestCudaAttentionBackendSelection(unittest.TestCase): "sglang.multimodal_gen.runtime.layers.attention.backends.sdpa.DynamicCudnnSDPABackend", ) + def test_blackwell_serves_fa4_head_sizes_outside_the_fa3_list(self): + FakeCudaPlatform.is_blackwell_device = True + fa_cls_str = "sglang.multimodal_gen.runtime.layers.attention.backends.flash_attn.FlashAttentionBackend" + dynamic_cls_str = "sglang.multimodal_gen.runtime.layers.attention.backends.sdpa.DynamicCudnnSDPABackend" + + with patch.object( + FakeCudaPlatform, + "_prepare_flash_attention_for_blackwell", + return_value=True, + ): + for head_size in (72, 80): + with self.subTest(head_size=head_size): + self.assertEqual( + FakeCudaPlatform.get_attn_backend_cls_str( + None, head_size, torch.bfloat16 + ), + dynamic_cls_str, + ) + self.assertEqual( + FakeCudaPlatform.get_attn_backend_cls_str( + AttentionBackendEnum.FA, head_size, torch.bfloat16 + ), + fa_cls_str, + ) + for head_size in (68, 136): + with self.subTest(head_size=head_size): + self.assertEqual( + FakeCudaPlatform.get_attn_backend_cls_str( + None, head_size, torch.bfloat16 + ), + SDPA_BACKEND_CLS_STR, + ) + + def test_pre_blackwell_keeps_torch_sdpa_for_head_size_72(self): + for selected_backend in (None, AttentionBackendEnum.FA): + with self.subTest(selected_backend=selected_backend): + self.assertEqual( + FakeCudaPlatform.get_attn_backend_cls_str( + selected_backend, 72, torch.bfloat16 + ), + SDPA_BACKEND_CLS_STR, + ) + def test_invalid_backend_raises(self): with self.assertRaisesRegex(ValueError, "Invalid attention backend"): self.resolve(AttentionBackendEnum.AITER_SAGE)