From 588049fe2eaad4bd41bf052a84b2110f52a1c331 Mon Sep 17 00:00:00 2001 From: Mick Date: Sun, 27 Sep 2026 22:54:46 +0800 Subject: [PATCH] [diffusion] fix: run an all-valid attention mask on the backend's unmasked kernel (#41309) Co-authored-by: Mick Qian Co-authored-by: Claude Opus 5.5 --- .../runtime/layers/attention/backends/sdpa.py | 2 +- .../runtime/layers/attention/layer.py | 11 ++ .../test/unit/test_usp_all_valid_key_mask.py | 137 ++++++++++++++++++ 3 files changed, 149 insertions(+), 1 deletion(-) create mode 100644 python/sglang/multimodal_gen/test/unit/test_usp_all_valid_key_mask.py diff --git a/python/sglang/multimodal_gen/runtime/layers/attention/backends/sdpa.py b/python/sglang/multimodal_gen/runtime/layers/attention/backends/sdpa.py index dc4fbca1bc..d185f9acec 100644 --- a/python/sglang/multimodal_gen/runtime/layers/attention/backends/sdpa.py +++ b/python/sglang/multimodal_gen/runtime/layers/attention/backends/sdpa.py @@ -309,7 +309,7 @@ class DynamicCudnnSDPAImpl(SDPAImpl): except RuntimeError as e: # cuDNN raises "No available kernel" for some shapes; pin the # FA fail-safe path for this layer and keep going. - logger.warning( + logger.warning_once( "cuDNN SDPA failed (%s); falling back to FlashAttention for %s.", e, type(self).__name__, diff --git a/python/sglang/multimodal_gen/runtime/layers/attention/layer.py b/python/sglang/multimodal_gen/runtime/layers/attention/layer.py index 7455bcc461..c1374fec86 100644 --- a/python/sglang/multimodal_gen/runtime/layers/attention/layer.py +++ b/python/sglang/multimodal_gen/runtime/layers/attention/layer.py @@ -1197,6 +1197,17 @@ class USPAttention(nn.Module): k = torch.cat([k_prefix, k], dim=1) v = torch.cat([v_prefix, v], dim=1) + # an all-valid key mask masks nothing; the dense SDPA mask below + # would pin a slow kernel (cutlassF on sm100) + if ( + attn_mask_meta is not None + and "indices" in attn_mask_meta + and attn_mask.dim() == 2 + and not torch.is_floating_point(attn_mask) + and attn_mask_meta["indices"].shape[0] == attn_mask.numel() + ): + return self.attn_impl.forward(q, k, v, ctx_attn_metadata) + q_ = q.transpose(1, 2) k_ = k.transpose(1, 2) v_ = v.transpose(1, 2) diff --git a/python/sglang/multimodal_gen/test/unit/test_usp_all_valid_key_mask.py b/python/sglang/multimodal_gen/test/unit/test_usp_all_valid_key_mask.py new file mode 100644 index 0000000000..5ed6ec8ef7 --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_usp_all_valid_key_mask.py @@ -0,0 +1,137 @@ +"""An all-valid key mask must not force the dense-mask SDPA path. + +Outside the FA backend, USPAttention serves a masked call with a dense SDPA +mask, which on sm100 pins PyTorch's cutlassF kernel even when the mask masks +nothing (Ideogram4 passes such a mask on every step). +""" + +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +import torch +import torch.nn.functional as F + +from sglang.multimodal_gen.runtime.layers.attention.layer import ( + USPAttention, + build_varlen_mask_meta, +) +from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum + +_LAYER = "sglang.multimodal_gen.runtime.layers.attention.layer" + + +class _CountingSdpa: + """Stands in for the backend kernel and counts calls.""" + + def __init__(self, scale: float): + self.scale = scale + self.calls = 0 + + def forward(self, q, k, v, _ctx): + self.calls += 1 + return _sdpa(q, k, v, self.scale) + + +def _sdpa(q, k, v, scale, key_mask=None): + return F.scaled_dot_product_attention( + q.transpose(1, 2), + k.transpose(1, 2), + v.transpose(1, 2), + attn_mask=None if key_mask is None else key_mask[:, None, None, :], + scale=scale, + ).transpose(1, 2) + + +class TestAllValidKeyMask(unittest.TestCase): + def setUp(self): + torch.manual_seed(0) + obj = USPAttention.__new__(USPAttention) + obj.causal = False + # sm100 default: the masked varlen fast path only serves FA + obj.backend = AttentionBackendEnum.DYNAMIC_CUDNN_SDPA + obj.softmax_scale = 8**-0.5 + obj.attn_impl = _CountingSdpa(obj.softmax_scale) + obj.allow_cudnn_sdp = False + obj.skip_sequence_parallel = False + obj.sp_attention_mode = "ulysses" + obj.sp_attention_mode_is_auto = False + self.attn = obj + + def _forward(self, *args, **kwargs): + with ( + patch( + f"{_LAYER}.get_forward_context", + return_value=SimpleNamespace(attn_metadata=None), + ), + patch(f"{_LAYER}.get_sequence_parallel_world_size", return_value=1), + ): + return self.attn.forward(*args, **kwargs) + + def test_all_valid_mask_runs_the_backend_kernel(self): + q, k, v = (torch.randn(2, 6, 2, 8) for _ in range(3)) + mask = torch.ones(2, 6, dtype=torch.bool) + + out = self._forward( + q, k, v, attn_mask=mask, attn_mask_meta=build_varlen_mask_meta(mask) + ) + + self.assertEqual(self.attn.attn_impl.calls, 1) + torch.testing.assert_close(out, _sdpa(q, k, v, self.attn.softmax_scale)) + + def test_padding_mask_keeps_the_masked_path(self): + q, k, v = (torch.randn(1, 6, 2, 8) for _ in range(3)) + mask = torch.ones(1, 6, dtype=torch.bool) + mask[:, 4:] = False + + out = self._forward( + q, k, v, attn_mask=mask, attn_mask_meta=build_varlen_mask_meta(mask) + ) + + self.assertEqual(self.attn.attn_impl.calls, 0) + torch.testing.assert_close( + out, _sdpa(q, k, v, self.attn.softmax_scale, key_mask=mask) + ) + + def test_segmented_prefix_reaches_the_backend_kernel_concatenated(self): + q, k, v = (torch.randn(1, 4, 2, 8) for _ in range(3)) + q_prefix, k_prefix, v_prefix = (torch.randn(1, 2, 2, 8) for _ in range(3)) + mask = torch.ones(1, 6, dtype=torch.bool) + + out = self._forward( + q, + k, + v, + attn_mask=mask, + attn_mask_meta=build_varlen_mask_meta(mask), + q_prefix=q_prefix, + k_prefix=k_prefix, + v_prefix=v_prefix, + ) + + self.assertEqual(self.attn.attn_impl.calls, 1) + expected = _sdpa( + torch.cat([q_prefix, q], dim=1), + torch.cat([k_prefix, k], dim=1), + torch.cat([v_prefix, v], dim=1), + self.attn.softmax_scale, + ) + torch.testing.assert_close(out, expected) + + def test_gap_meta_without_indices_keeps_the_masked_path(self): + q, k, v = (torch.randn(1, 6, 2, 8) for _ in range(3)) + mask = torch.ones(1, 6, dtype=torch.bool) + mask[:, 4:] = False + + out = self._forward( + q, k, v, attn_mask=mask, attn_mask_meta={"pad_start": 4, "pad_end": 6} + ) + + self.assertEqual(self.attn.attn_impl.calls, 0) + torch.testing.assert_close( + out, _sdpa(q, k, v, self.attn.softmax_scale, key_mask=mask) + ) + + +if __name__ == "__main__": + unittest.main()