From bf8adf96024799843faff49e33bf758518893408 Mon Sep 17 00:00:00 2001 From: saatwiknagpal Date: Thu, 1 Oct 2026 20:22:53 -0700 Subject: [PATCH] [diffusion] fix: fix int32 offset overflow in SubBlock router kernels (#41965) --- .../backends/subblock_sparse/kernels.py | 15 ++++++++------- .../unit/test_subblock_sparse_attention.py | 19 +++++++++++++++++++ 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/python/sglang/multimodal_gen/runtime/layers/attention/backends/subblock_sparse/kernels.py b/python/sglang/multimodal_gen/runtime/layers/attention/backends/subblock_sparse/kernels.py index 17a71ed4e7..8de3d732f3 100644 --- a/python/sglang/multimodal_gen/runtime/layers/attention/backends/subblock_sparse/kernels.py +++ b/python/sglang/multimodal_gen/runtime/layers/attention/backends/subblock_sparse/kernels.py @@ -53,6 +53,8 @@ def _score_kernel( pid_m = tl.program_id(0) pid_n = tl.program_id(1) pid_l = tl.program_id(2) + # int64: element offsets pass 2^31 for long sequences with many heads per call. + O += pid_l.to(tl.int64) * stride_ol offs_m = pid_m * BLK_M + tl.arange(0, BLK_M) offs_n = pid_n * BLK_N + tl.arange(0, BLK_N) @@ -102,10 +104,7 @@ def _score_kernel( offs_o = pid_n * (BLK_N // NK) + tl.arange(0, BLK_N // NK) offs_q = pid_m * (BLK_M // NQR) + tl.arange(0, BLK_M // NQR) tl.store( - O - + pid_l * stride_ol - + offs_q[:, None] * stride_om - + offs_o[None, :] * stride_on, + O + offs_q[:, None] * stride_om + offs_o[None, :] * stride_on, out, mask=(offs_q[:, None] < MOUT) & (offs_o[None, :] < NOUT), ) @@ -175,10 +174,11 @@ def _pool_kernel( bf16; the PyTorch version needs an fp32 temporary plus a transpose. """ cell = tl.program_id(0) - l = tl.program_id(1) + # int64: element offsets pass 2^31 for long sequences with many heads per call. + l = tl.program_id(1).to(tl.int64) b = l // H h = l % H - offs_t = cell * SUB + tl.arange(0, SUB) + offs_t = cell.to(tl.int64) * SUB + tl.arange(0, SUB) offs_d = tl.arange(0, D) mask = offs_t < S x = tl.load( @@ -236,7 +236,8 @@ def _topk_kernel(S, OUT, G, K, BLK: tl.constexpr, ITERS: tl.constexpr): up with a different set than an exact top-K, but only among blocks that tie at the threshold -- the total selected score differs by ~1e-6 relative, which is nothing. """ - row = tl.program_id(0) + # int64: element offsets pass 2^31 for long sequences with many heads per call. + row = tl.program_id(0).to(tl.int64) offs = tl.arange(0, BLK) m = offs < G s = tl.load(S + row * G + offs, mask=m, other=-float("inf")).to(tl.float32) diff --git a/python/sglang/multimodal_gen/test/unit/test_subblock_sparse_attention.py b/python/sglang/multimodal_gen/test/unit/test_subblock_sparse_attention.py index 05dcb8f252..5712fa6b2f 100644 --- a/python/sglang/multimodal_gen/test/unit/test_subblock_sparse_attention.py +++ b/python/sglang/multimodal_gen/test/unit/test_subblock_sparse_attention.py @@ -577,5 +577,24 @@ class TestSubBlockNumerics(unittest.TestCase): self.assertGreater(_cosine(out[start:stop], ref), 0.999) +@unittest.skipUnless( + torch.cuda.is_available() and torch.cuda.mem_get_info()[0] >= 48 * 2**30, + "needs a CUDA GPU with 48 GiB free", +) +class TestRouterPastInt32(unittest.TestCase): + def test_last_heads_match_when_tensors_pass_int32(self): + """With q/k and the score matrix past 2^31 elements, the last heads must + still match the same heads routed alone.""" + shape = (1, 491_520, 56, HEAD_DIM) + q = torch.randn(shape, device="cuda", dtype=torch.bfloat16) + k = torch.randn_like(q) + router, tail = SubBlockRouter(), slice(54, 56) + kwargs = dict(sparsity=0.8, softmax_scale=HEAD_DIM**-0.5) + index = router.route(q, k, **kwargs).index[:, tail].sort(dim=-1).values + q, k = q[:, :, tail].contiguous(), k[:, :, tail].contiguous() + ref = router.route(q, k, **kwargs).index.sort(dim=-1).values + self.assertTrue(torch.equal(index, ref)) + + if __name__ == "__main__": unittest.main()