diff --git a/python/sglang/kernels/ops/diffusion/layout/ulysses_qkv_triton.py b/python/sglang/kernels/ops/diffusion/layout/ulysses_qkv_triton.py index c330e48f17..67b72089b2 100644 --- a/python/sglang/kernels/ops/diffusion/layout/ulysses_qkv_triton.py +++ b/python/sglang/kernels/ops/diffusion/layout/ulysses_qkv_triton.py @@ -6,6 +6,9 @@ import triton.language as tl _MAX_INT32 = 2**31 - 1 +# Devices with a Triton backend for this kernel. "cuda" also covers ROCm. +_SUPPORTED_DEVICE_TYPES = ("cuda", "xpu") + @triton.jit(do_not_specialize=["rows"]) def _pack_qkv_destination_major_kernel( @@ -70,8 +73,11 @@ def pack_qkv_destination_major( """Pack matching ``[rows, global_heads, head_size]`` Q/K/V tensors.""" if q.dim() != 3 or q.shape != k.shape or q.shape != v.shape: raise ValueError("q, k, and v must have the same 3D shape") - if not (q.is_cuda and k.is_cuda and v.is_cuda): - raise ValueError("q, k, and v must be CUDA tensors") + if not all(t.device.type in _SUPPORTED_DEVICE_TYPES for t in (q, k, v)): + raise ValueError( + f"q, k, and v must be on one of {_SUPPORTED_DEVICE_TYPES}, " + f"got {q.device.type}" + ) if not (q.device == k.device == v.device and q.dtype == k.dtype == v.dtype): raise ValueError("q, k, and v must have the same device and dtype") if q.stride(-1) != 1 or k.stride(-1) != 1 or v.stride(-1) != 1: diff --git a/python/sglang/multimodal_gen/runtime/layers/usp.py b/python/sglang/multimodal_gen/runtime/layers/usp.py index 151fe0d2dc..80f9213050 100644 --- a/python/sglang/multimodal_gen/runtime/layers/usp.py +++ b/python/sglang/multimodal_gen/runtime/layers/usp.py @@ -52,6 +52,11 @@ def uncached_a2a_staging(): _UNCACHED_STAGING -= 1 +# Accelerators whose device module exposes is_current_stream_capturing() and can +# run the packed-QKV relayout kernel. "cuda" also covers ROCm, where is_cuda is True. +_A2A_DEVICE_TYPES = ("cuda", "xpu") + + def drop_a2a_staging_buffers() -> None: """Release the cached all-to-all staging buffers on this rank. @@ -61,8 +66,9 @@ def drop_a2a_staging_buffers() -> None: """ if not _A2A_STAGING_BUFFERS: return - if torch.cuda.is_available(): - torch.cuda.synchronize() + device_module = torch.get_device_module() + if device_module.is_available(): + device_module.synchronize() _A2A_STAGING_BUFFERS.clear() @@ -75,23 +81,23 @@ def _a2a_staging_buffer( next collective with the same role overwrites it. Keep one grow-only backing allocation per (role, dtype, device), then return an exact-shape view into that allocation. This removes per-block allocator churn without - retaining one CUDA tensor for every request shape seen by the worker. - Bypassed under autograd and CUDA graph capture: a buffer first allocated - while capturing would live in the graph's private memory pool and must - not be shared with eager replays. + retaining one device tensor for every request shape seen by the worker. + Bypassed under autograd and graph capture: a buffer first allocated while + capturing would live in the graph's private memory pool and must not be + shared with eager replays. """ if ( _UNCACHED_STAGING or torch.is_grad_enabled() or torch.compiler.is_compiling() - or device.type != "cuda" - or torch.cuda.is_current_stream_capturing() + or device.type not in _A2A_DEVICE_TYPES + or torch.get_device_module(device).is_current_stream_capturing() ): return torch.empty(shape, dtype=dtype, device=device) device_index = device.index if device_index is None: - device_index = torch.cuda.current_device() + device_index = torch.get_device_module(device).current_device() key = (role, dtype, device_index) required_numel = math.prod(shape) buffer = _A2A_STAGING_BUFFERS.get(key) @@ -417,7 +423,7 @@ def _usp_input_all_to_all_packed_qkv( h_local = h_global // world_size if ( - q.is_cuda + q.device.type in _A2A_DEVICE_TYPES and q.dtype in (torch.float16, torch.bfloat16) and q.dtype == k.dtype == v.dtype and q.stride(-1) == k.stride(-1) == v.stride(-1) == 1 @@ -469,7 +475,7 @@ def _can_use_packed_qkv_a2a_4d( q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, world_size: int ) -> bool: return ( - q.is_cuda + q.device.type in _A2A_DEVICE_TYPES and q.ndim == 4 and q.shape == k.shape == v.shape and q.dtype == k.dtype == v.dtype diff --git a/python/sglang/multimodal_gen/test/unit/test_minimax_h3_dit_contract.py b/python/sglang/multimodal_gen/test/unit/test_minimax_h3_dit_contract.py index 6738dc81cb..c3fc844913 100644 --- a/python/sglang/multimodal_gen/test/unit/test_minimax_h3_dit_contract.py +++ b/python/sglang/multimodal_gen/test/unit/test_minimax_h3_dit_contract.py @@ -51,6 +51,7 @@ from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum from sglang.multimodal_gen.test.single_test_file.component_accuracy.utils import ( ensure_distributed_env_defaults, ) +from sglang.srt.utils import get_device def _ensure_single_process_parallel_runtime() -> None: @@ -817,16 +818,20 @@ def test_packed_qkv_exchange_preserves_rank_and_head_order(_): torch.testing.assert_close(actual[index], expected) -@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") -def test_cuda_ulysses_qkv_pack_is_bit_exact(): +@pytest.mark.skipif( + not (torch.cuda.is_available() or torch.xpu.is_available()), + reason="requires CUDA or XPU", +) +def test_ulysses_qkv_pack_is_bit_exact(): from sglang.kernels.ops.diffusion import pack_qkv_destination_major + device = get_device() torch.manual_seed(23) rows, world_size, heads, head_size = 65, 8, 56, 128 qkv = torch.randn( rows, 3 * heads * head_size, - device="cuda", + device=device, dtype=torch.bfloat16, ) q, k, v = ( @@ -839,7 +844,7 @@ def test_cuda_ulysses_qkv_pack_is_bit_exact(): rows, local_heads, 3 * head_size, - device="cuda", + device=device, dtype=torch.bfloat16, ) for index, tensor in enumerate((q, k, v)): diff --git a/python/sglang/multimodal_gen/test/unit/test_usp_packed_qkv_a2a.py b/python/sglang/multimodal_gen/test/unit/test_usp_packed_qkv_a2a.py index f9f966c698..daca32e9a3 100644 --- a/python/sglang/multimodal_gen/test/unit/test_usp_packed_qkv_a2a.py +++ b/python/sglang/multimodal_gen/test/unit/test_usp_packed_qkv_a2a.py @@ -1,7 +1,8 @@ """The packed Ulysses Q/K/V input exchange must be bit-identical to the unpacked path. The collective is emulated in-process with exact ``all_to_all_single`` chunk semantics (rank r's j-th chunk goes to rank j's -r-th chunk); the pack kernel and unpack views run unmodified on CUDA.""" +r-th chunk); the pack kernel and unpack views run unmodified on the +accelerator.""" import math import unittest @@ -10,15 +11,22 @@ from unittest.mock import patch import torch from sglang.multimodal_gen.runtime.layers import usp as usp_mod +from sglang.srt.utils import get_device from sglang.test.test_utils import CustomTestCase _USP = "sglang.multimodal_gen.runtime.layers.usp" -@unittest.skipUnless(torch.cuda.is_available(), "requires CUDA") +@unittest.skipIf( + not (torch.cuda.is_available() or torch.xpu.is_available()), + "requires CUDA or XPU", +) class TestA2AStagingBuffer(CustomTestCase): def setUp(self): super().setUp() + # Indexed: the staging-buffer cache key holds device.index. + device_module = torch.get_device_module() + self.device = torch.device(get_device(device_module.current_device())) usp_mod._A2A_STAGING_BUFFERS.clear() def tearDown(self): @@ -26,7 +34,7 @@ class TestA2AStagingBuffer(CustomTestCase): super().tearDown() def test_cache_capacity_is_bounded_across_shapes(self): - device = torch.device("cuda", torch.cuda.current_device()) + device = self.device role = "test_role" shapes = ((2, 3), (4, 5), (5, 4), (1, 7), (3, 11), (2, 4)) @@ -60,7 +68,7 @@ class TestA2AStagingBuffer(CustomTestCase): ) def test_smaller_shape_reuses_larger_backing_buffer(self): - device = torch.device("cuda", torch.cuda.current_device()) + device = self.device with torch.no_grad(): large = usp_mod._a2a_staging_buffer( @@ -74,7 +82,7 @@ class TestA2AStagingBuffer(CustomTestCase): self.assertEqual(small.shape, (2, 7)) def test_role_and_dtype_are_separate_cache_keys(self): - device = torch.device("cuda", torch.cuda.current_device()) + device = self.device with torch.no_grad(): usp_mod._a2a_staging_buffer("input", (8,), torch.float16, device) @@ -84,32 +92,36 @@ class TestA2AStagingBuffer(CustomTestCase): self.assertEqual(len(usp_mod._A2A_STAGING_BUFFERS), 3) def test_bypass_paths_do_not_replace_cached_storage(self): - cuda_device = torch.device("cuda", torch.cuda.current_device()) + device = self.device with torch.no_grad(): cached = usp_mod._a2a_staging_buffer( - "test_role", (8,), torch.float16, cuda_device + "test_role", (8,), torch.float16, device ) - key = ("test_role", torch.float16, cuda_device.index) + key = ("test_role", torch.float16, device.index) cached_storage = usp_mod._A2A_STAGING_BUFFERS[key] with torch.enable_grad(): grad_buffer = usp_mod._a2a_staging_buffer( - "test_role", (16,), torch.float16, cuda_device + "test_role", (16,), torch.float16, device ) with ( torch.no_grad(), patch(f"{_USP}.torch.compiler.is_compiling", return_value=True), ): compile_buffer = usp_mod._a2a_staging_buffer( - "test_role", (16,), torch.float16, cuda_device + "test_role", (16,), torch.float16, device ) with ( torch.no_grad(), - patch(f"{_USP}.torch.cuda.is_current_stream_capturing", return_value=True), + patch.object( + torch.get_device_module(device), + "is_current_stream_capturing", + return_value=True, + ), ): capture_buffer = usp_mod._a2a_staging_buffer( - "test_role", (16,), torch.float16, cuda_device + "test_role", (16,), torch.float16, device ) with torch.no_grad(): cpu_buffer = usp_mod._a2a_staging_buffer( @@ -119,13 +131,16 @@ class TestA2AStagingBuffer(CustomTestCase): self.assertEqual(list(usp_mod._A2A_STAGING_BUFFERS), [key]) self.assertIs(usp_mod._A2A_STAGING_BUFFERS[key], cached_storage) self.assertEqual(cached.numel(), 8) - self.assertEqual(grad_buffer.device.type, "cuda") - self.assertEqual(compile_buffer.device.type, "cuda") - self.assertEqual(capture_buffer.device.type, "cuda") + self.assertEqual(grad_buffer.device.type, device.type) + self.assertEqual(compile_buffer.device.type, device.type) + self.assertEqual(capture_buffer.device.type, device.type) self.assertEqual(cpu_buffer.device.type, "cpu") -@unittest.skipUnless(torch.cuda.is_available(), "requires CUDA") +@unittest.skipIf( + not (torch.cuda.is_available() or torch.xpu.is_available()), + "requires CUDA or XPU", +) class TestPackedQKVInputA2A(CustomTestCase): def _run_all_ranks(self, fn, world): sends, recvs = [], None @@ -154,7 +169,7 @@ class TestPackedQKVInputA2A(CustomTestCase): s_local, h_local = s_global // world, h_global // world full = [ torch.randn( - b, s_global, h_global, d, dtype=torch.bfloat16, device="cuda" + b, s_global, h_global, d, dtype=torch.bfloat16, device=get_device() ) for _ in range(3) ] @@ -173,8 +188,44 @@ class TestPackedQKVInputA2A(CustomTestCase): ) self.assertTrue(packed[r][i].is_contiguous()) + def test_packed_path_issues_one_collective(self): + """The unpacked fallback is also bit-exact, so the equivalence test above + passes either way; only the collective count proves the packed path ran. + """ + world, b, s_global, h_global, d = 4, 1, 128, 8, 64 + shard = tuple( + torch.randn( + b, + s_global // world, + h_global, + d, + dtype=torch.bfloat16, + device=get_device(), + ) + for _ in range(3) + ) + collectives = 0 -@unittest.skipUnless(torch.cuda.is_available(), "requires CUDA") + def counting_a2a(x, role=None): + nonlocal collectives + collectives += 1 + return torch.empty_like(x) + + with ( + torch.no_grad(), + patch(f"{_USP}._usp_all_to_all_single", counting_a2a), + patch(f"{_USP}.get_ulysses_parallel_world_size", return_value=world), + ): + self.assertTrue(usp_mod._can_use_packed_qkv_a2a_4d(*shard, world)) + usp_mod._usp_input_all_to_all_qkv(*shard) + + self.assertEqual(collectives, 1) + + +@unittest.skipIf( + not (torch.cuda.is_available() or torch.xpu.is_available()), + "requires CUDA or XPU", +) class TestPackedQKVRowViewPredicate(CustomTestCase): """`_packed_qkv_row_view_is_free` decides which path the a2a takes. @@ -194,7 +245,7 @@ class TestPackedQKVRowViewPredicate(CustomTestCase): case this PR exists to accept. """ inner = self.H * self.D - hidden = torch.randn(b, s, 3 * inner, device="cuda", dtype=torch.bfloat16) + hidden = torch.randn(b, s, 3 * inner, device=get_device(), dtype=torch.bfloat16) q, k, v = hidden.chunk(3, dim=-1) return tuple(t.unflatten(-1, (self.H, self.D)) for t in (q, k, v)) @@ -210,7 +261,9 @@ class TestPackedQKVRowViewPredicate(CustomTestCase): def test_batched_sequence_slice_is_rejected(self): """b > 1 sliced along seq: stride(0) still spans the unsliced rows, so merging batch and seq would copy. Must stay on the unpacked path.""" - full = torch.randn(2, 64, self.H, self.D, device="cuda", dtype=torch.bfloat16) + full = torch.randn( + 2, 64, self.H, self.D, device=get_device(), dtype=torch.bfloat16 + ) sl = full[:, 32:] self.assertEqual(sl.stride(-1), 1, "setup: head_size is unit stride") self.assertNotEqual(sl.stride(0), sl.shape[1] * sl.stride(1)) @@ -219,7 +272,9 @@ class TestPackedQKVRowViewPredicate(CustomTestCase): def test_contiguous_still_accepted(self): """No regression for the layouts that already qualified.""" - t = torch.randn(1, 32, self.H, self.D, device="cuda", dtype=torch.bfloat16) + t = torch.randn( + 1, 32, self.H, self.D, device=get_device(), dtype=torch.bfloat16 + ) self.assertTrue(t.is_contiguous()) self.assertTrue(usp_mod._packed_qkv_row_view_is_free(t)) self.assertTrue(usp_mod._can_use_packed_qkv_a2a_4d(t, t, t, self.WORLD)) @@ -227,7 +282,7 @@ class TestPackedQKVRowViewPredicate(CustomTestCase): def test_non_unit_head_stride_is_rejected(self): """head_size must be unit-stride; the kernel cannot express a gap.""" wide = torch.randn( - 1, 32, self.H, 2 * self.D, device="cuda", dtype=torch.bfloat16 + 1, 32, self.H, 2 * self.D, device=get_device(), dtype=torch.bfloat16 ) t = wide[..., ::2] self.assertNotEqual(t.stride(-1), 1)