From 6e7beace143964386ce2e4418401b7369f0dabd0 Mon Sep 17 00:00:00 2001 From: saatwiknagpal Date: Thu, 8 Oct 2026 20:51:46 -0700 Subject: [PATCH] [diffusion] fix: fix int32 offset overflow in the Ulysses packed-QKV pack kernel (#43140) --- .../diffusion/layout/ulysses_qkv_triton.py | 20 +++++++++++++++- .../kernels/ops/diffusion/test_layout.py | 24 +++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/python/sglang/kernels/ops/diffusion/layout/ulysses_qkv_triton.py b/python/sglang/kernels/ops/diffusion/layout/ulysses_qkv_triton.py index e341124da3..c330e48f17 100644 --- a/python/sglang/kernels/ops/diffusion/layout/ulysses_qkv_triton.py +++ b/python/sglang/kernels/ops/diffusion/layout/ulysses_qkv_triton.py @@ -4,6 +4,8 @@ import torch import triton import triton.language as tl +_MAX_INT32 = 2**31 - 1 + @triton.jit(do_not_specialize=["rows"]) def _pack_qkv_destination_major_kernel( @@ -21,9 +23,15 @@ def _pack_qkv_destination_major_kernel( stride_k_head, stride_v_row, stride_v_head, + IDX64: tl.constexpr, BLOCK_SIZE: tl.constexpr, ): - offsets = tl.program_id(0) * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE) + if IDX64: + offsets = tl.program_id(0).to(tl.int64) * BLOCK_SIZE + tl.arange( + 0, BLOCK_SIZE + ).to(tl.int64) + else: + offsets = tl.program_id(0) * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE) mask = offsets < total_elements dim = offsets % head_size @@ -96,6 +104,15 @@ def pack_qkv_destination_major( if total_elements == 0: return output + # Input views may be strided (e.g. slices of a fused qkv projection), so + # bound each tensor by one past the largest element offset it is addressed at. + max_offset = max( + output.numel(), + *( + (rows - 1) * t.stride(0) + (global_heads - 1) * t.stride(1) + head_size + for t in (q, k, v) + ), + ) block_size = 1024 with torch.get_device_module().device(q.device): _pack_qkv_destination_major_kernel[(triton.cdiv(total_elements, block_size),)]( @@ -113,6 +130,7 @@ def pack_qkv_destination_major( k.stride(1), v.stride(0), v.stride(1), + IDX64=max_offset > _MAX_INT32, BLOCK_SIZE=block_size, num_warps=8, ) diff --git a/test/registered/kernels/ops/diffusion/test_layout.py b/test/registered/kernels/ops/diffusion/test_layout.py index 4e41ccdee0..b9b9c3a2db 100644 --- a/test/registered/kernels/ops/diffusion/test_layout.py +++ b/test/registered/kernels/ops/diffusion/test_layout.py @@ -166,6 +166,30 @@ def test_pack_qkv_destination_major_validates_inputs(): pack_qkv_destination_major(q, q, q, 2, out=torch.empty_like(q)) +@pytest.mark.skipif( + not torch.cuda.is_available() or torch.cuda.mem_get_info()[0] < 24 * 2**30, + reason="needs a CUDA GPU with 24 GiB free", +) +def test_pack_qkv_destination_major_rows_past_int32(): + """Past 2^31 elements, the last rows must pack exactly as the same rows + packed alone, whether the packed output, the q/k/v views, or both pass it.""" + rows, heads, head_size = 119_024, 56, 128 + qkv = torch.randn(rows, 3 * heads * head_size, device="cuda", dtype=torch.bfloat16) + views = [ + t.view(rows, heads, head_size) for t in qkv.split(heads * head_size, dim=-1) + ] + tail = slice(100_000, None) + cases = ( + ([t.contiguous() for t in views], 4), # only the output passes 2^31 + (views, 4), # both pass 2^31 + ([t[:, :1] for t in views], 1), # only the q/k/v views pass 2^31 + ) + for (q, k, v), world in cases: + out = pack_qkv_destination_major(q, k, v, world)[:, tail] + ref = pack_qkv_destination_major(q[tail], k[tail], v[tail], world) + assert torch.equal(out, ref) + + # --------------------------------------------------------------------------- # Varlen pack / scatter # ---------------------------------------------------------------------------