[diffusion] fix: run an all-valid attention mask on the backend's unmasked kernel (#41309)

Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Mick
2026-09-27 22:54:46 +08:00
committed by GitHub
co-authored by Mick Qian Claude Opus 5.5
parent 84622ce9d5
commit 588049fe2e
3 changed files with 149 additions and 1 deletions
@@ -309,7 +309,7 @@ class DynamicCudnnSDPAImpl(SDPAImpl):
except RuntimeError as e:
# cuDNN raises "No available kernel" for some shapes; pin the
# FA fail-safe path for this layer and keep going.
logger.warning(
logger.warning_once(
"cuDNN SDPA failed (%s); falling back to FlashAttention for %s.",
e,
type(self).__name__,
@@ -1197,6 +1197,17 @@ class USPAttention(nn.Module):
k = torch.cat([k_prefix, k], dim=1)
v = torch.cat([v_prefix, v], dim=1)
# an all-valid key mask masks nothing; the dense SDPA mask below
# would pin a slow kernel (cutlassF on sm100)
if (
attn_mask_meta is not None
and "indices" in attn_mask_meta
and attn_mask.dim() == 2
and not torch.is_floating_point(attn_mask)
and attn_mask_meta["indices"].shape[0] == attn_mask.numel()
):
return self.attn_impl.forward(q, k, v, ctx_attn_metadata)
q_ = q.transpose(1, 2)
k_ = k.transpose(1, 2)
v_ = v.transpose(1, 2)
@@ -0,0 +1,137 @@
"""An all-valid key mask must not force the dense-mask SDPA path.
Outside the FA backend, USPAttention serves a masked call with a dense SDPA
mask, which on sm100 pins PyTorch's cutlassF kernel even when the mask masks
nothing (Ideogram4 passes such a mask on every step).
"""
import unittest
from types import SimpleNamespace
from unittest.mock import patch
import torch
import torch.nn.functional as F
from sglang.multimodal_gen.runtime.layers.attention.layer import (
USPAttention,
build_varlen_mask_meta,
)
from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum
_LAYER = "sglang.multimodal_gen.runtime.layers.attention.layer"
class _CountingSdpa:
"""Stands in for the backend kernel and counts calls."""
def __init__(self, scale: float):
self.scale = scale
self.calls = 0
def forward(self, q, k, v, _ctx):
self.calls += 1
return _sdpa(q, k, v, self.scale)
def _sdpa(q, k, v, scale, key_mask=None):
return F.scaled_dot_product_attention(
q.transpose(1, 2),
k.transpose(1, 2),
v.transpose(1, 2),
attn_mask=None if key_mask is None else key_mask[:, None, None, :],
scale=scale,
).transpose(1, 2)
class TestAllValidKeyMask(unittest.TestCase):
def setUp(self):
torch.manual_seed(0)
obj = USPAttention.__new__(USPAttention)
obj.causal = False
# sm100 default: the masked varlen fast path only serves FA
obj.backend = AttentionBackendEnum.DYNAMIC_CUDNN_SDPA
obj.softmax_scale = 8**-0.5
obj.attn_impl = _CountingSdpa(obj.softmax_scale)
obj.allow_cudnn_sdp = False
obj.skip_sequence_parallel = False
obj.sp_attention_mode = "ulysses"
obj.sp_attention_mode_is_auto = False
self.attn = obj
def _forward(self, *args, **kwargs):
with (
patch(
f"{_LAYER}.get_forward_context",
return_value=SimpleNamespace(attn_metadata=None),
),
patch(f"{_LAYER}.get_sequence_parallel_world_size", return_value=1),
):
return self.attn.forward(*args, **kwargs)
def test_all_valid_mask_runs_the_backend_kernel(self):
q, k, v = (torch.randn(2, 6, 2, 8) for _ in range(3))
mask = torch.ones(2, 6, dtype=torch.bool)
out = self._forward(
q, k, v, attn_mask=mask, attn_mask_meta=build_varlen_mask_meta(mask)
)
self.assertEqual(self.attn.attn_impl.calls, 1)
torch.testing.assert_close(out, _sdpa(q, k, v, self.attn.softmax_scale))
def test_padding_mask_keeps_the_masked_path(self):
q, k, v = (torch.randn(1, 6, 2, 8) for _ in range(3))
mask = torch.ones(1, 6, dtype=torch.bool)
mask[:, 4:] = False
out = self._forward(
q, k, v, attn_mask=mask, attn_mask_meta=build_varlen_mask_meta(mask)
)
self.assertEqual(self.attn.attn_impl.calls, 0)
torch.testing.assert_close(
out, _sdpa(q, k, v, self.attn.softmax_scale, key_mask=mask)
)
def test_segmented_prefix_reaches_the_backend_kernel_concatenated(self):
q, k, v = (torch.randn(1, 4, 2, 8) for _ in range(3))
q_prefix, k_prefix, v_prefix = (torch.randn(1, 2, 2, 8) for _ in range(3))
mask = torch.ones(1, 6, dtype=torch.bool)
out = self._forward(
q,
k,
v,
attn_mask=mask,
attn_mask_meta=build_varlen_mask_meta(mask),
q_prefix=q_prefix,
k_prefix=k_prefix,
v_prefix=v_prefix,
)
self.assertEqual(self.attn.attn_impl.calls, 1)
expected = _sdpa(
torch.cat([q_prefix, q], dim=1),
torch.cat([k_prefix, k], dim=1),
torch.cat([v_prefix, v], dim=1),
self.attn.softmax_scale,
)
torch.testing.assert_close(out, expected)
def test_gap_meta_without_indices_keeps_the_masked_path(self):
q, k, v = (torch.randn(1, 6, 2, 8) for _ in range(3))
mask = torch.ones(1, 6, dtype=torch.bool)
mask[:, 4:] = False
out = self._forward(
q, k, v, attn_mask=mask, attn_mask_meta={"pad_start": 4, "pad_end": 6}
)
self.assertEqual(self.attn.attn_impl.calls, 0)
torch.testing.assert_close(
out, _sdpa(q, k, v, self.attn.softmax_scale, key_mask=mask)
)
if __name__ == "__main__":
unittest.main()