[Perf] [XPU] usp: Enable packed Ulysses QKV all-to-all (#39989)

Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Chetan Kumar Verma
2026-10-16 10:08:50 +08:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 4e85e8ed37
commit 1e33d2c4bd
4 changed files with 111 additions and 39 deletions
@@ -6,6 +6,9 @@ import triton.language as tl
_MAX_INT32 = 2**31 - 1
# Devices with a Triton backend for this kernel. "cuda" also covers ROCm.
_SUPPORTED_DEVICE_TYPES = ("cuda", "xpu")
@triton.jit(do_not_specialize=["rows"])
def _pack_qkv_destination_major_kernel(
@@ -70,8 +73,11 @@ def pack_qkv_destination_major(
"""Pack matching ``[rows, global_heads, head_size]`` Q/K/V tensors."""
if q.dim() != 3 or q.shape != k.shape or q.shape != v.shape:
raise ValueError("q, k, and v must have the same 3D shape")
if not (q.is_cuda and k.is_cuda and v.is_cuda):
raise ValueError("q, k, and v must be CUDA tensors")
if not all(t.device.type in _SUPPORTED_DEVICE_TYPES for t in (q, k, v)):
raise ValueError(
f"q, k, and v must be on one of {_SUPPORTED_DEVICE_TYPES}, "
f"got {q.device.type}"
)
if not (q.device == k.device == v.device and q.dtype == k.dtype == v.dtype):
raise ValueError("q, k, and v must have the same device and dtype")
if q.stride(-1) != 1 or k.stride(-1) != 1 or v.stride(-1) != 1:
@@ -52,6 +52,11 @@ def uncached_a2a_staging():
_UNCACHED_STAGING -= 1
# Accelerators whose device module exposes is_current_stream_capturing() and can
# run the packed-QKV relayout kernel. "cuda" also covers ROCm, where is_cuda is True.
_A2A_DEVICE_TYPES = ("cuda", "xpu")
def drop_a2a_staging_buffers() -> None:
"""Release the cached all-to-all staging buffers on this rank.
@@ -61,8 +66,9 @@ def drop_a2a_staging_buffers() -> None:
"""
if not _A2A_STAGING_BUFFERS:
return
if torch.cuda.is_available():
torch.cuda.synchronize()
device_module = torch.get_device_module()
if device_module.is_available():
device_module.synchronize()
_A2A_STAGING_BUFFERS.clear()
@@ -75,23 +81,23 @@ def _a2a_staging_buffer(
next collective with the same role overwrites it. Keep one grow-only
backing allocation per (role, dtype, device), then return an exact-shape
view into that allocation. This removes per-block allocator churn without
retaining one CUDA tensor for every request shape seen by the worker.
Bypassed under autograd and CUDA graph capture: a buffer first allocated
while capturing would live in the graph's private memory pool and must
not be shared with eager replays.
retaining one device tensor for every request shape seen by the worker.
Bypassed under autograd and graph capture: a buffer first allocated while
capturing would live in the graph's private memory pool and must not be
shared with eager replays.
"""
if (
_UNCACHED_STAGING
or torch.is_grad_enabled()
or torch.compiler.is_compiling()
or device.type != "cuda"
or torch.cuda.is_current_stream_capturing()
or device.type not in _A2A_DEVICE_TYPES
or torch.get_device_module(device).is_current_stream_capturing()
):
return torch.empty(shape, dtype=dtype, device=device)
device_index = device.index
if device_index is None:
device_index = torch.cuda.current_device()
device_index = torch.get_device_module(device).current_device()
key = (role, dtype, device_index)
required_numel = math.prod(shape)
buffer = _A2A_STAGING_BUFFERS.get(key)
@@ -417,7 +423,7 @@ def _usp_input_all_to_all_packed_qkv(
h_local = h_global // world_size
if (
q.is_cuda
q.device.type in _A2A_DEVICE_TYPES
and q.dtype in (torch.float16, torch.bfloat16)
and q.dtype == k.dtype == v.dtype
and q.stride(-1) == k.stride(-1) == v.stride(-1) == 1
@@ -469,7 +475,7 @@ def _can_use_packed_qkv_a2a_4d(
q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, world_size: int
) -> bool:
return (
q.is_cuda
q.device.type in _A2A_DEVICE_TYPES
and q.ndim == 4
and q.shape == k.shape == v.shape
and q.dtype == k.dtype == v.dtype
@@ -51,6 +51,7 @@ from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum
from sglang.multimodal_gen.test.single_test_file.component_accuracy.utils import (
ensure_distributed_env_defaults,
)
from sglang.srt.utils import get_device
def _ensure_single_process_parallel_runtime() -> None:
@@ -817,16 +818,20 @@ def test_packed_qkv_exchange_preserves_rank_and_head_order(_):
torch.testing.assert_close(actual[index], expected)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_cuda_ulysses_qkv_pack_is_bit_exact():
@pytest.mark.skipif(
not (torch.cuda.is_available() or torch.xpu.is_available()),
reason="requires CUDA or XPU",
)
def test_ulysses_qkv_pack_is_bit_exact():
from sglang.kernels.ops.diffusion import pack_qkv_destination_major
device = get_device()
torch.manual_seed(23)
rows, world_size, heads, head_size = 65, 8, 56, 128
qkv = torch.randn(
rows,
3 * heads * head_size,
device="cuda",
device=device,
dtype=torch.bfloat16,
)
q, k, v = (
@@ -839,7 +844,7 @@ def test_cuda_ulysses_qkv_pack_is_bit_exact():
rows,
local_heads,
3 * head_size,
device="cuda",
device=device,
dtype=torch.bfloat16,
)
for index, tensor in enumerate((q, k, v)):
@@ -1,7 +1,8 @@
"""The packed Ulysses Q/K/V input exchange must be bit-identical to the
unpacked path. The collective is emulated in-process with exact
``all_to_all_single`` chunk semantics (rank r's j-th chunk goes to rank j's
r-th chunk); the pack kernel and unpack views run unmodified on CUDA."""
r-th chunk); the pack kernel and unpack views run unmodified on the
accelerator."""
import math
import unittest
@@ -10,15 +11,22 @@ from unittest.mock import patch
import torch
from sglang.multimodal_gen.runtime.layers import usp as usp_mod
from sglang.srt.utils import get_device
from sglang.test.test_utils import CustomTestCase
_USP = "sglang.multimodal_gen.runtime.layers.usp"
@unittest.skipUnless(torch.cuda.is_available(), "requires CUDA")
@unittest.skipIf(
not (torch.cuda.is_available() or torch.xpu.is_available()),
"requires CUDA or XPU",
)
class TestA2AStagingBuffer(CustomTestCase):
def setUp(self):
super().setUp()
# Indexed: the staging-buffer cache key holds device.index.
device_module = torch.get_device_module()
self.device = torch.device(get_device(device_module.current_device()))
usp_mod._A2A_STAGING_BUFFERS.clear()
def tearDown(self):
@@ -26,7 +34,7 @@ class TestA2AStagingBuffer(CustomTestCase):
super().tearDown()
def test_cache_capacity_is_bounded_across_shapes(self):
device = torch.device("cuda", torch.cuda.current_device())
device = self.device
role = "test_role"
shapes = ((2, 3), (4, 5), (5, 4), (1, 7), (3, 11), (2, 4))
@@ -60,7 +68,7 @@ class TestA2AStagingBuffer(CustomTestCase):
)
def test_smaller_shape_reuses_larger_backing_buffer(self):
device = torch.device("cuda", torch.cuda.current_device())
device = self.device
with torch.no_grad():
large = usp_mod._a2a_staging_buffer(
@@ -74,7 +82,7 @@ class TestA2AStagingBuffer(CustomTestCase):
self.assertEqual(small.shape, (2, 7))
def test_role_and_dtype_are_separate_cache_keys(self):
device = torch.device("cuda", torch.cuda.current_device())
device = self.device
with torch.no_grad():
usp_mod._a2a_staging_buffer("input", (8,), torch.float16, device)
@@ -84,32 +92,36 @@ class TestA2AStagingBuffer(CustomTestCase):
self.assertEqual(len(usp_mod._A2A_STAGING_BUFFERS), 3)
def test_bypass_paths_do_not_replace_cached_storage(self):
cuda_device = torch.device("cuda", torch.cuda.current_device())
device = self.device
with torch.no_grad():
cached = usp_mod._a2a_staging_buffer(
"test_role", (8,), torch.float16, cuda_device
"test_role", (8,), torch.float16, device
)
key = ("test_role", torch.float16, cuda_device.index)
key = ("test_role", torch.float16, device.index)
cached_storage = usp_mod._A2A_STAGING_BUFFERS[key]
with torch.enable_grad():
grad_buffer = usp_mod._a2a_staging_buffer(
"test_role", (16,), torch.float16, cuda_device
"test_role", (16,), torch.float16, device
)
with (
torch.no_grad(),
patch(f"{_USP}.torch.compiler.is_compiling", return_value=True),
):
compile_buffer = usp_mod._a2a_staging_buffer(
"test_role", (16,), torch.float16, cuda_device
"test_role", (16,), torch.float16, device
)
with (
torch.no_grad(),
patch(f"{_USP}.torch.cuda.is_current_stream_capturing", return_value=True),
patch.object(
torch.get_device_module(device),
"is_current_stream_capturing",
return_value=True,
),
):
capture_buffer = usp_mod._a2a_staging_buffer(
"test_role", (16,), torch.float16, cuda_device
"test_role", (16,), torch.float16, device
)
with torch.no_grad():
cpu_buffer = usp_mod._a2a_staging_buffer(
@@ -119,13 +131,16 @@ class TestA2AStagingBuffer(CustomTestCase):
self.assertEqual(list(usp_mod._A2A_STAGING_BUFFERS), [key])
self.assertIs(usp_mod._A2A_STAGING_BUFFERS[key], cached_storage)
self.assertEqual(cached.numel(), 8)
self.assertEqual(grad_buffer.device.type, "cuda")
self.assertEqual(compile_buffer.device.type, "cuda")
self.assertEqual(capture_buffer.device.type, "cuda")
self.assertEqual(grad_buffer.device.type, device.type)
self.assertEqual(compile_buffer.device.type, device.type)
self.assertEqual(capture_buffer.device.type, device.type)
self.assertEqual(cpu_buffer.device.type, "cpu")
@unittest.skipUnless(torch.cuda.is_available(), "requires CUDA")
@unittest.skipIf(
not (torch.cuda.is_available() or torch.xpu.is_available()),
"requires CUDA or XPU",
)
class TestPackedQKVInputA2A(CustomTestCase):
def _run_all_ranks(self, fn, world):
sends, recvs = [], None
@@ -154,7 +169,7 @@ class TestPackedQKVInputA2A(CustomTestCase):
s_local, h_local = s_global // world, h_global // world
full = [
torch.randn(
b, s_global, h_global, d, dtype=torch.bfloat16, device="cuda"
b, s_global, h_global, d, dtype=torch.bfloat16, device=get_device()
)
for _ in range(3)
]
@@ -173,8 +188,44 @@ class TestPackedQKVInputA2A(CustomTestCase):
)
self.assertTrue(packed[r][i].is_contiguous())
def test_packed_path_issues_one_collective(self):
"""The unpacked fallback is also bit-exact, so the equivalence test above
passes either way; only the collective count proves the packed path ran.
"""
world, b, s_global, h_global, d = 4, 1, 128, 8, 64
shard = tuple(
torch.randn(
b,
s_global // world,
h_global,
d,
dtype=torch.bfloat16,
device=get_device(),
)
for _ in range(3)
)
collectives = 0
@unittest.skipUnless(torch.cuda.is_available(), "requires CUDA")
def counting_a2a(x, role=None):
nonlocal collectives
collectives += 1
return torch.empty_like(x)
with (
torch.no_grad(),
patch(f"{_USP}._usp_all_to_all_single", counting_a2a),
patch(f"{_USP}.get_ulysses_parallel_world_size", return_value=world),
):
self.assertTrue(usp_mod._can_use_packed_qkv_a2a_4d(*shard, world))
usp_mod._usp_input_all_to_all_qkv(*shard)
self.assertEqual(collectives, 1)
@unittest.skipIf(
not (torch.cuda.is_available() or torch.xpu.is_available()),
"requires CUDA or XPU",
)
class TestPackedQKVRowViewPredicate(CustomTestCase):
"""`_packed_qkv_row_view_is_free` decides which path the a2a takes.
@@ -194,7 +245,7 @@ class TestPackedQKVRowViewPredicate(CustomTestCase):
case this PR exists to accept.
"""
inner = self.H * self.D
hidden = torch.randn(b, s, 3 * inner, device="cuda", dtype=torch.bfloat16)
hidden = torch.randn(b, s, 3 * inner, device=get_device(), dtype=torch.bfloat16)
q, k, v = hidden.chunk(3, dim=-1)
return tuple(t.unflatten(-1, (self.H, self.D)) for t in (q, k, v))
@@ -210,7 +261,9 @@ class TestPackedQKVRowViewPredicate(CustomTestCase):
def test_batched_sequence_slice_is_rejected(self):
"""b > 1 sliced along seq: stride(0) still spans the unsliced rows, so
merging batch and seq would copy. Must stay on the unpacked path."""
full = torch.randn(2, 64, self.H, self.D, device="cuda", dtype=torch.bfloat16)
full = torch.randn(
2, 64, self.H, self.D, device=get_device(), dtype=torch.bfloat16
)
sl = full[:, 32:]
self.assertEqual(sl.stride(-1), 1, "setup: head_size is unit stride")
self.assertNotEqual(sl.stride(0), sl.shape[1] * sl.stride(1))
@@ -219,7 +272,9 @@ class TestPackedQKVRowViewPredicate(CustomTestCase):
def test_contiguous_still_accepted(self):
"""No regression for the layouts that already qualified."""
t = torch.randn(1, 32, self.H, self.D, device="cuda", dtype=torch.bfloat16)
t = torch.randn(
1, 32, self.H, self.D, device=get_device(), dtype=torch.bfloat16
)
self.assertTrue(t.is_contiguous())
self.assertTrue(usp_mod._packed_qkv_row_view_is_free(t))
self.assertTrue(usp_mod._can_use_packed_qkv_a2a_4d(t, t, t, self.WORLD))
@@ -227,7 +282,7 @@ class TestPackedQKVRowViewPredicate(CustomTestCase):
def test_non_unit_head_stride_is_rejected(self):
"""head_size must be unit-stride; the kernel cannot express a gap."""
wide = torch.randn(
1, 32, self.H, 2 * self.D, device="cuda", dtype=torch.bfloat16
1, 32, self.H, 2 * self.D, device=get_device(), dtype=torch.bfloat16
)
t = wide[..., ::2]
self.assertNotEqual(t.stride(-1), 1)