diff --git a/python/sglang/kernels/ops/attention/dsv4/decode_attention_sm100.py b/python/sglang/kernels/ops/attention/dsv4/decode_attention_sm100.py index c8e85fa119..ef5f2e29eb 100644 --- a/python/sglang/kernels/ops/attention/dsv4/decode_attention_sm100.py +++ b/python/sglang/kernels/ops/attention/dsv4/decode_attention_sm100.py @@ -11,6 +11,8 @@ from .kv_layout import KVLayout LAYOUT = KVLayout.V4 MAX_BATCH = 8 +# SM100 row cap, below the ~48-row break-even against FlashMLA. +MAX_ROWS = 40 NUM_HEADS = 16 HEAD_DIM = 512 SOFTMAX_SCALE = HEAD_DIM**-0.5 @@ -26,7 +28,7 @@ def can_use_swapab_attention( ) -> bool: """The caller adds the SM100 and single-query forward-mode gates.""" return ( - 0 < q.shape[0] <= MAX_BATCH + 0 < q.shape[0] <= MAX_ROWS and num_heads in (NUM_HEADS, 2 * NUM_HEADS) and q.dtype == torch.bfloat16 and q.shape[-1] == HEAD_DIM @@ -119,7 +121,7 @@ def swapab_attention( block = 64 assert q.ndim in (3, 4) and (q.ndim == 3 or q.shape[1] == 1) - assert 0 < b <= MAX_BATCH and d == HEAD_DIM + assert 0 < b <= MAX_ROWS and d == HEAD_DIM assert q.dtype == torch.bfloat16 and q.stride(-1) == 1 assert kv.shape[-1] == LAYOUT.bytes_per_token assert kv.dtype in (torch.uint8, torch.float8_e4m3fn) diff --git a/test/registered/kernels/ops/attention/dsv4/test_swapab_attention.py b/test/registered/kernels/ops/attention/dsv4/test_swapab_attention.py index acc7c22894..6cba4dccfb 100644 --- a/test/registered/kernels/ops/attention/dsv4/test_swapab_attention.py +++ b/test/registered/kernels/ops/attention/dsv4/test_swapab_attention.py @@ -100,13 +100,13 @@ class TestSwapABAttention(CustomTestCase): ) ei = ei_storage[..., :ne] lengths = torch.tensor( - [0, 1, 31, 63, 64, nk - 1, nk, nk + 3][:b], device="cuda", dtype=torch.int32 - ) + [0, 1, 31, 63, 64, nk - 1, nk, nk + 3] * 5, device="cuda", dtype=torch.int32 + )[:b] el = torch.tensor( - [ne, ne // 2, 1, 0, 63, ne - 1, 65, ne][:b], + [ne, ne // 2, 1, 0, 63, ne - 1, 65, ne] * 5, device="cuda", dtype=torch.int32, - ) + )[:b] sink = torch.randn(64, device="cuda") for h0 in range(0, heads, 16): sink[h0], sink[h0 + 1], sink[h0 + 2] = -torch.inf, torch.inf, 20.0 @@ -234,6 +234,14 @@ class TestSwapABAttention(CustomTestCase): self.check_case(*case, heads=32) self.check_case(6, 64, 192, 512, graph=True, heads=32) + @unittest.skipIf(is_hip(), "the gfx950 kernel keeps the 8-row cap") + def test_rows_up_to_cap(self): + """SM100 admits up to 40 rows: DSpark verify at TP4 bs <= 8.""" + for case in ((20, 16, 133, 0), (40, 128, 192, 1024)): + with self.subTest(case=case): + self.check_case(*case) + self.check_case(40, 64, 192, 512, graph=True) + if __name__ == "__main__": unittest.main()